Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(managed-migrations): stop admin saves overwriting worker state (#118356)

Eli Reisman20e721143090
diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py
index 6da852b85108..4f4a556f5677 100644
--- a/posthog/models/activity_logging/activity_log.py
+++ b/posthog/models/activity_logging/activity_log.py
@@ -1007,6 +1007,7 @@ field_exclusions: dict[AuditableScope, list[str]] = {
"lease_id",
"backoff_attempt",
"backoff_until",
+ "pending_commit",
],
"Integration": [
"sensitive_config",
diff --git a/products/managed_migrations/backend/admin/batch_imports.py b/products/managed_migrations/backend/admin/batch_imports.py
index c8081f23a728..630c001db5f4 100644
--- a/products/managed_migrations/backend/admin/batch_imports.py
+++ b/products/managed_migrations/backend/admin/batch_imports.py
@@ -1,18 +1,24 @@
+from typing import cast
+
from django import forms
from django.contrib import admin, messages
from django.utils.html import format_html
-from products.managed_migrations.backend.models.batch_imports import DEFAULT_SEND_RATE, BatchImport
+from products.managed_migrations.backend.models.batch_imports import DEFAULT_SEND_RATE, BatchImport, CommitInFlightError
+
+STATUS_CHANGED_MESSAGE = "This import changed since the page loaded. Reload it and try again."
+SINK_TYPE_CHOICES: list[tuple[str, str]] = [("capture", "Capture"), ("kafka", "Kafka")]
class BatchImportAdminForm(forms.ModelForm):
sink_type = forms.ChoiceField(
- choices=[("capture", "Capture"), ("kafka", "Kafka")],
+ choices=SINK_TYPE_CHOICES,
required=False,
help_text="Capture sends events via HTTP; Kafka writes directly to Kafka.",
)
send_rate = forms.IntegerField(
required=False,
+ min_value=1,
help_text=f"Events per second to send (e.g., {DEFAULT_SEND_RATE}). Leave empty to keep current value.",
widget=forms.NumberInput(attrs={"placeholder": f"e.g., {DEFAULT_SEND_RATE}"}),
)
@@ -23,30 +29,75 @@ class BatchImportAdminForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ # Compare submitted values with what the page showed, not the row at save time, so a
+ # worker write after page load is not counted as an operator edit.
+ for field in self.fields.values():
+ field.show_hidden_initial = True
if self.instance and self.instance.import_config:
sink = self.instance.import_config.get("sink", {})
- if sink.get("type"):
- self.fields["sink_type"].initial = sink["type"]
+ sink_type = sink.get("type")
+ if sink_type:
+ self.fields["sink_type"].initial = sink_type
+ # Without its own choice, a trial or noop sink would submit "capture" and be
+ # rewritten on every save.
+ if sink_type not in dict(SINK_TYPE_CHOICES):
+ cast(forms.ChoiceField, self.fields["sink_type"]).choices = [
+ *SINK_TYPE_CHOICES,
+ (sink_type, f"{sink_type} (current)"),
+ ]
if sink.get("send_rate"):
self.fields["send_rate"].initial = sink["send_rate"]
+ def clean(self):
+ cleaned_data = super().clean()
+ # A page rendered without hidden initial values would make every field read as changed.
+ if self.instance.pk and any(self.add_initial_prefix(name) not in self.data for name in self.fields):
+ raise forms.ValidationError("This page is out of date. Reload it and make your change again.")
+ if self.resumes():
+ # Django admin runs the whole POST in one transaction, so this lock holds until save_model
+ # writes the resume. A refusal here re-renders the form with the error and saves nothing.
+ self.raise_if_resume_unsafe(BatchImport.objects.select_for_update().get(pk=self.instance.pk))
+ return cleaned_data
+
+ def resumes(self) -> bool:
+ return (
+ bool(self.instance.pk)
+ and "status" in self.changed_data
+ and self.cleaned_data.get("status") == BatchImport.Status.RUNNING
+ )
+
+ def raise_if_resume_unsafe(self, current: BatchImport) -> None:
+ # A resume clears the lease, so it must act on the status the operator saw, not one that a
+ # worker or a customer set since.
+ if current.status != self.data.get(self.add_initial_prefix("status")):
+ raise forms.ValidationError(STATUS_CHANGED_MESSAGE)
+ if current.commit_in_flight():
+ raise forms.ValidationError(str(CommitInFlightError()))
+
def save(self, commit=True):
instance = super().save(commit=False)
sink_type = self.cleaned_data.get("sink_type")
send_rate = self.cleaned_data.get("send_rate")
- if sink_type:
+ if sink_type and any(name in self.changed_data for name in ("sink_type", "send_rate")):
if not instance.import_config:
instance.import_config = {}
existing_sink = instance.import_config.get("sink", {})
- sink = {
- "type": sink_type,
- "send_rate": send_rate if send_rate is not None else existing_sink.get("send_rate", DEFAULT_SEND_RATE),
- }
- if sink_type == "kafka":
- sink["topic"] = existing_sink.get("topic", "historical")
- sink["transaction_timeout_seconds"] = existing_sink.get("transaction_timeout_seconds", 60)
+ if sink_type == existing_sink.get("type"):
+ sink = {**existing_sink}
+ if send_rate is not None:
+ sink["send_rate"] = send_rate
+ else:
+ sink = {
+ "type": sink_type,
+ "send_rate": send_rate
+ if send_rate is not None
+ else existing_sink.get("send_rate", DEFAULT_SEND_RATE),
+ }
+ if sink_type == "kafka":
+ sink["topic"] = existing_sink.get("topic", "historical")
+ sink["transaction_timeout_seconds"] = existing_sink.get("transaction_timeout_seconds", 60)
instance.import_config["sink"] = sink
@@ -55,6 +106,10 @@ class BatchImportAdminForm(forms.ModelForm):
return instance
+EDITABLE_MODEL_FIELDS = ("team", "created_by_id", "status", "status_message")
+SINK_FORM_FIELDS = ("sink_type", "send_rate")
+
+
@admin.register(BatchImport)
class BatchImportAdmin(admin.ModelAdmin):
form = BatchImportAdminForm
@@ -83,6 +138,7 @@ class BatchImportAdmin(admin.ModelAdmin):
"leased_until",
"backoff_attempt",
"backoff_until",
+ "pending_commit",
"get_worker_progress",
)
autocomplete_fields = ("team",)
@@ -97,12 +153,14 @@ class BatchImportAdmin(admin.ModelAdmin):
"leased_until",
"backoff_attempt",
"backoff_until",
+ "pending_commit",
),
"description": (
- "A paused job keeps its worker lease; both resume actions clear it "
- "(a bare status change leaves the row unclaimable until the lease expires, "
- "up to 30 minutes). A running job with a future leased_until is actively "
- "claimed by a worker. Choosing a resume action: 'Resume (keep progress)' "
+ "A running job with a future leased_until is actively claimed by a worker. "
+ "Changing the status to running, or using a resume action, clears the lease, backoff "
+ "and pending commit so a worker can claim the job at once. A pending commit on "
+ "a job no worker holds means a worker stopped mid-commit: the chunk it names "
+ "may or may not have reached the sink. Choosing a resume action: 'Resume (keep progress)' "
"continues from the saved byte offset - right for transient pauses and "
"for source data that has not changed. 'Resume + re-import in-flight part' "
"resets the current part to offset 0 first - right when the source bytes "
@@ -131,6 +189,22 @@ class BatchImportAdmin(admin.ModelAdmin):
("Metadata", {"fields": ("id", "created_at", "updated_at", "state")}),
)
+ def save_model(self, request, obj, form, change):
+ if not change:
+ super().save_model(request, obj, form, change)
+ return
+
+ # A full-row save would overwrite the progress, lease and backoff that a running worker
+ # saved after the page loaded.
+ update_fields = [name for name in form.changed_data if name in EDITABLE_MODEL_FIELDS]
+ if any(name in form.changed_data for name in SINK_FORM_FIELDS):
+ update_fields.append("import_config")
+ if form.resumes():
+ message = obj.status_message if "status_message" in form.changed_data else "Resumed by admin"
+ update_fields.extend(obj.reset_for_resume(message))
+ if update_fields:
+ obj.save(update_fields=list(dict.fromkeys([*update_fields, "updated_at"])))
+
@admin.display(description="Progress")
def get_progress(self, obj):
"""Parts done / total, from the worker-owned state JSON."""
diff --git a/products/managed_migrations/backend/api/batch_imports.py b/products/managed_migrations/backend/api/batch_imports.py
index 1ebf532ea572..7ae5407ce56f 100644
--- a/products/managed_migrations/backend/api/batch_imports.py
+++ b/products/managed_migrations/backend/api/batch_imports.py
@@ -32,6 +32,7 @@ from products.managed_migrations.backend.models.batch_imports import (
DEFAULT_SEND_RATE,
BatchImport,
BatchImportConfigBuilder,
+ CommitInFlightError,
ContentType,
DateRangeExportSource,
get_aws_external_id,
@@ -892,27 +893,13 @@ class BatchImportViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
"""Resume a paused batch import."""
batch_import = self.get_object()
- if batch_import.status != BatchImport.Status.PAUSED:
+ try:
+ batch_import.resume_from_pause("Resumed by user")
+ except CommitInFlightError as e:
+ return Response({"error": str(e)}, status=status.HTTP_409_CONFLICT)
+ except ValueError:
return Response({"error": "Only paused imports can be resumed"}, status=status.HTTP_400_BAD_REQUEST)
- batch_import.status = BatchImport.Status.RUNNING
- batch_import.status_message = "Resumed by user"
- batch_import.lease_id = None
- batch_import.leased_until = None
- batch_import.backoff_attempt = 0
- batch_import.backoff_until = None
- batch_import.save(
- update_fields=[
- "status",
- "status_message",
- "lease_id",
- "leased_until",
- "backoff_attempt",
- "backoff_until",
- "updated_at",
- ]
- )
-
return Response({"status": "resumed"})
@extend_schema(
diff --git a/products/managed_migrations/backend/api/test/test_batch_imports.py b/products/managed_migrations/backend/api/test/test_batch_imports.py
index a6359106db0c..ab6536eee363 100644
--- a/products/managed_migrations/backend/api/test/test_batch_imports.py
+++ b/products/managed_migrations/backend/api/test/test_batch_imports.py
@@ -3,15 +3,19 @@ from datetime import UTC, datetime, timedelta
from posthog.test.base import APIBaseTest, BaseTest
from unittest.mock import patch
+from django.contrib import admin
from django.contrib.admin.sites import AdminSite
from django.contrib.messages import get_messages
from django.contrib.messages.middleware import MessageMiddleware
from django.contrib.sessions.middleware import SessionMiddleware
+from django.db import connection
from django.http import HttpResponse
from django.test import RequestFactory, SimpleTestCase, override_settings
+from django.test.utils import CaptureQueriesContext
from parameterized import parameterized
+from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.security.url_validation import UNREACHABLE_HOST_MESSAGE
from products.managed_migrations.backend import trial_storage
@@ -21,6 +25,7 @@ from products.managed_migrations.backend.models.batch_imports import (
DEFAULT_SEND_RATE,
BatchImport,
BatchImportConfigBuilder,
+ CommitInFlightError,
ContentType,
get_aws_external_id,
)
@@ -55,9 +60,10 @@ class TestBatchImportModel(BaseTest):
status_message="parse error",
display_status_message="Invalid JSON syntax",
lease_id="worker-lease",
- leased_until=datetime.now(UTC) + timedelta(minutes=25),
+ leased_until=datetime.now(UTC) - timedelta(minutes=1),
backoff_attempt=3,
backoff_until=datetime.now(UTC) + timedelta(minutes=5),
+ pending_commit={"key": "day-2", "from_offset": 8999000, "to_offset": 8999445},
state=state,
)
@@ -150,21 +156,27 @@ class TestBatchImportModel(BaseTest):
self.assertEqual(batch_import.backoff_attempt, 0)
self.assertIsNone(batch_import.backoff_until)
self.assertIsNone(batch_import.display_status_message)
+ self.assertIsNone(batch_import.pending_commit)
def test_resume_after_pause_keeps_progress_and_clears_lease(self):
batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})
+ stale_copy = BatchImport.objects.get(id=batch_import.id)
+ worker_progress = {
+ "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}]
+ }
+ BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress)
- batch_import.resume_after_pause()
+ stale_copy.resume_after_pause()
batch_import.refresh_from_db()
- assert batch_import.state is not None
- self.assertEqual(batch_import.state["parts"], self.MIXED_PROGRESS_PARTS)
+ self.assertEqual(batch_import.state, worker_progress)
self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
self.assertIsNone(batch_import.lease_id)
self.assertIsNone(batch_import.leased_until)
self.assertEqual(batch_import.backoff_attempt, 0)
self.assertIsNone(batch_import.backoff_until)
self.assertIsNone(batch_import.display_status_message)
+ self.assertIsNone(batch_import.pending_commit)
@parameterized.expand(
[
@@ -187,6 +199,49 @@ class TestBatchImportModel(BaseTest):
self.assertEqual(batch_import.state["parts"][1]["current_offset"], 8999445)
self.assertEqual(batch_import.lease_id, "worker-lease")
+ @parameterized.expand(
+ [
+ (f"{method}_{case}", method, row, refused)
+ for method in ("resume_after_pause", "resume_with_inflight_part_reset")
+ for case, row, refused in (
+ ("marker_and_live_lease", {"leased_until": timedelta(minutes=5)}, True),
+ (
+ "legacy_commit_message_and_live_lease",
+ {
+ "leased_until": timedelta(minutes=5),
+ "pending_commit": None,
+ "status_message": "Starting commit of part day-2 to offset 9, consumed 9 additional bytes",
+ },
+ True,
+ ),
+ ("marker_and_expired_lease", {"leased_until": -timedelta(minutes=1)}, False),
+ ("marker_and_no_lease", {"lease_id": None, "leased_until": None}, False),
+ )
+ ]
+ )
+ def test_resume_refuses_while_a_worker_sends_a_chunk(self, _name, method, row, refused):
+ batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})
+ if isinstance(row.get("leased_until"), timedelta):
+ row = {**row, "leased_until": datetime.now(UTC) + row["leased_until"]}
+ BatchImport.objects.filter(id=batch_import.id).update(**row)
+
+ if refused:
+ with self.assertRaises(CommitInFlightError):
+ getattr(BatchImport.objects.get(id=batch_import.id), method)()
+ else:
+ getattr(BatchImport.objects.get(id=batch_import.id), method)()
+
+ batch_import.refresh_from_db()
+ if refused:
+ assert batch_import.state is not None
+ self.assertEqual(batch_import.status, BatchImport.Status.PAUSED)
+ self.assertIsNotNone(batch_import.lease_id)
+ self.assertEqual(batch_import.state["parts"][1]["current_offset"], 8999445)
+ else:
+ self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
+ self.assertIsNone(batch_import.lease_id)
+ self.assertIsNone(batch_import.pending_commit)
+
@parameterized.expand(
[
("all_parts_done", {"parts": [{"key": "day-1", "current_offset": 1000, "total_size": 1000}]}),
@@ -605,6 +660,201 @@ class TestBatchImportAdminActions(BaseTest):
self.assertEqual(levels, ["success", "warning"])
+class TestBatchImportAdminChangeForm(BaseTest):
+ WORKER_STATE = {"parts": [{"key": "day-1", "current_offset": 9000, "total_size": None}]}
+
+ def _form_values(self, batch_import: BatchImport) -> dict[str, str]:
+ sink = batch_import.import_config["sink"]
+ return {
+ "team": str(batch_import.team_id),
+ "created_by_id": str(batch_import.created_by_id),
+ "status": batch_import.status,
+ "status_message": batch_import.status_message or "",
+ "sink_type": sink["type"],
+ "send_rate": str(sink.get("send_rate", "")),
+ }
+
+ def _post_admin_form(
+ self, batch_import: BatchImport, shown: dict[str, str], edits: dict[str, str], with_initial=True
+ ):
+ self.user.is_staff = True
+ self.user.save()
+ self.model_admin = admin.site._registry[BatchImport]
+ self.request = RequestFactory().post("/")
+ self.request.user = self.user
+ # Django admin loads the row when the POST arrives, so it already holds any worker write.
+ loaded_at_post = BatchImport.objects.get(id=batch_import.id)
+ initial = {f"initial-{name}": value for name, value in shown.items()} if with_initial else {}
+ return self.model_admin.get_form(self.request, loaded_at_post, change=True)(
+ data={**initial, **shown, **edits}, instance=loaded_at_post
+ )
+
+ def _save(self, form) -> None:
+ self.model_admin.save_model(self.request, form.save(commit=False), form, change=True)
+
+ @parameterized.expand(
+ [
+ ("trial", {"type": "trial_s3", "record_limit": 100}, {}, {"type": "trial_s3", "record_limit": 100}),
+ ("noop", {"type": "noop"}, {}, {"type": "noop"}),
+ (
+ "trial_rate_change_keeps_record_limit",
+ {"type": "trial_s3", "record_limit": 100},
+ {"send_rate": "600"},
+ {"type": "trial_s3", "record_limit": 100, "send_rate": 600},
+ ),
+ ]
+ )
+ def test_save_changes_only_the_sink_keys_the_operator_edits(self, _name, sink, edits, expected_sink):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": sink},
+ status=BatchImport.Status.PAUSED,
+ )
+
+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status_message": "note", **edits})
+ self.assertTrue(form.is_valid(), form.errors)
+ self._save(form)
+
+ batch_import.refresh_from_db()
+ self.assertEqual(batch_import.import_config["sink"], expected_sink)
+ self.assertEqual(batch_import.status_message, "note")
+
+ def test_save_refuses_a_resume_while_a_worker_sends_a_chunk(self):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ status=BatchImport.Status.PAUSED,
+ lease_id="live-worker-lease",
+ leased_until=datetime.now(UTC) + timedelta(minutes=5),
+ pending_commit={"key": "day-1", "from_offset": 0, "to_offset": 10},
+ )
+
+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})
+
+ self.assertFalse(form.is_valid())
+ self.assertIn("A worker is sending a chunk", str(form.non_field_errors()))
+
+ @parameterized.expand(
+ [
+ ("resumed_and_running", BatchImport.Status.RUNNING),
+ ("completed", BatchImport.Status.COMPLETED),
+ ("failed", BatchImport.Status.FAILED),
+ ]
+ )
+ def test_save_refuses_a_resume_when_the_status_changed_since_page_load(self, _name, current_status):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ status=BatchImport.Status.PAUSED,
+ )
+ shown = self._form_values(batch_import)
+ BatchImport.objects.filter(id=batch_import.id).update(
+ status=current_status,
+ lease_id="live-worker-lease",
+ leased_until=datetime.now(UTC) + timedelta(minutes=5),
+ )
+
+ form = self._post_admin_form(batch_import, shown, {"status": "running"})
+
+ self.assertFalse(form.is_valid())
+ self.assertIn("changed since the page loaded", str(form.non_field_errors()))
+
+ @parameterized.expand(
+ [
+ (
+ "send_started",
+ {"pending_commit": {"key": "day-1", "from_offset": 0, "to_offset": 10}},
+ "A worker is sending a chunk",
+ ),
+ ("resumed_elsewhere", {"status": BatchImport.Status.RUNNING}, "changed since the page loaded"),
+ ]
+ )
+ def test_save_refuses_a_resume_when_the_row_changes_after_the_admin_loads_it(self, _name, change, expected_error):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ status=BatchImport.Status.PAUSED,
+ )
+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})
+ BatchImport.objects.filter(id=batch_import.id).update(
+ lease_id="live-worker-lease", leased_until=datetime.now(UTC) + timedelta(minutes=5), **change
+ )
+
+ with CaptureQueriesContext(connection) as queries:
+ self.assertFalse(form.is_valid())
+
+ self.assertIn(expected_error, str(form.non_field_errors()))
+ self.assertTrue(
+ any("FOR UPDATE" in q["sql"] and "posthog_batchimport" in q["sql"] for q in queries.captured_queries)
+ )
+
+ def test_save_rejects_a_page_without_initial_values(self):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ status=BatchImport.Status.RUNNING,
+ lease_id="live-worker-lease",
+ )
+
+ form = self._post_admin_form(
+ batch_import, self._form_values(batch_import), {"send_rate": "500"}, with_initial=False
+ )
+
+ self.assertFalse(form.is_valid())
+ self.assertIn("This page is out of date", str(form.non_field_errors()))
+
+ @parameterized.expand(
+ [
+ ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500),
+ ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000),
+ ("operator_resume", "paused", "paused", {"status": "running"}, "running", 5000),
+ ]
+ )
+ def test_save_writes_only_operator_edits(
+ self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate
+ ):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ secrets={"secret": "value"},
+ status=shown_status,
+ lease_id="lease-at-page-load",
+ leased_until=datetime.now(UTC) - timedelta(minutes=1),
+ backoff_attempt=2,
+ pending_commit={"key": "day-1", "from_offset": 8000, "to_offset": 9000},
+ state={"parts": [{"key": "day-1", "current_offset": 100, "total_size": None}]},
+ )
+ shown = self._form_values(batch_import)
+ BatchImport.objects.filter(id=batch_import.id).update(
+ status=worker_status, state=self.WORKER_STATE, lease_id="lease-after-worker-commit"
+ )
+
+ form = self._post_admin_form(batch_import, shown, edits)
+ self.assertTrue(form.is_valid(), form.errors)
+ self._save(form)
+
+ batch_import.refresh_from_db()
+ self.assertEqual(batch_import.status, expected_status)
+ self.assertEqual(batch_import.import_config["sink"]["send_rate"], expected_send_rate)
+ self.assertEqual(batch_import.state, self.WORKER_STATE)
+ if expected_status == "running":
+ self.assertEqual(batch_import.status_message, "Resumed by admin")
+ self.assertIsNone(batch_import.lease_id)
+ self.assertIsNone(batch_import.leased_until)
+ self.assertEqual(batch_import.backoff_attempt, 0)
+ self.assertIsNone(batch_import.pending_commit)
+ else:
+ self.assertEqual(batch_import.lease_id, "lease-after-worker-commit")
+ self.assertEqual(batch_import.backoff_attempt, 2)
+ self.assertIsNotNone(batch_import.pending_commit)
+
+
class TestBatchImportAPI(APIBaseTest):
def test_model_creation_only(self):
batch_import = BatchImport.objects.create(
@@ -1062,6 +1312,25 @@ class TestBatchImportAPI(APIBaseTest):
self.assertEqual(results[0]["id"], str(batch_import.id))
self.assertEqual(results[0]["display_status"], expected_display_status)
+ def test_resume_refuses_while_a_worker_sends_a_chunk(self):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"source": {"type": "s3"}},
+ status=BatchImport.Status.PAUSED,
+ lease_id="live-worker-lease",
+ leased_until=datetime.now(tz=UTC) + timedelta(minutes=5),
+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},
+ )
+
+ response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")
+
+ self.assertEqual(response.status_code, 409)
+ batch_import.refresh_from_db()
+ self.assertEqual(batch_import.status, BatchImport.Status.PAUSED)
+ self.assertEqual(batch_import.lease_id, "live-worker-lease")
+ self.assertIsNotNone(batch_import.pending_commit)
+
def test_resume_clears_lease_and_backoff(self):
batch_import = BatchImport.objects.create(
team=self.team,
@@ -1070,9 +1339,11 @@ class TestBatchImportAPI(APIBaseTest):
secrets={"access_key": "test"},
status=BatchImport.Status.PAUSED,
lease_id="old-lease-uuid",
- leased_until=datetime.now(tz=UTC) + timedelta(hours=1),
+ leased_until=datetime.now(tz=UTC) - timedelta(minutes=1),
backoff_attempt=5,
backoff_until=datetime.now(tz=UTC) + timedelta(hours=1),
+ display_status_message="Job paused after a failed commit",
+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},
)
response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")
@@ -1085,6 +1356,15 @@ class TestBatchImportAPI(APIBaseTest):
self.assertEqual(batch_import.backoff_attempt, 0)
self.assertIsNone(batch_import.backoff_until)
self.assertEqual(batch_import.status_message, "Resumed by user")
+ self.assertIsNone(batch_import.display_status_message)
+ self.assertIsNone(batch_import.pending_commit)
+ logged_fields = [
+ change["field"]
+ for log in ActivityLog.objects.filter(scope="BatchImport", item_id=str(batch_import.id))
+ for change in (log.detail or {}).get("changes") or []
+ ]
+ self.assertIn("status", logged_fields)
+ self.assertNotIn("pending_commit", logged_fields)
@parameterized.expand(
[
diff --git a/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.py b/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.py
new file mode 100644
index 000000000000..90e1111a5a95
--- /dev/null
+++ b/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.2.18 on 2026-10-16 07:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("managed_migrations", "0003_alter_batchimport_team"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="batchimport",
+ name="pending_commit",
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/products/managed_migrations/backend/migrations/max_migration.txt b/products/managed_migrations/backend/migrations/max_migration.txt
index 0e41e10acac8..26518050ee84 100644
--- a/products/managed_migrations/backend/migrations/max_migration.txt
+++ b/products/managed_migrations/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0003_alter_batchimport_team
+0004_batchimport_pending_commit
diff --git a/products/managed_migrations/backend/models/batch_imports.py b/products/managed_migrations/backend/models/batch_imports.py
index 8acc298df552..11db6f563624 100644
--- a/products/managed_migrations/backend/models/batch_imports.py
+++ b/products/managed_migrations/backend/models/batch_imports.py
@@ -2,7 +2,8 @@ from enum import Enum
from typing import TYPE_CHECKING, Self
from django.conf import settings
-from django.db import models
+from django.db import models, transaction
+from django.utils import timezone
from posthog.helpers.encrypted_fields import EncryptedJSONStringField
from posthog.models.activity_logging.model_activity import ModelActivityMixin
@@ -16,6 +17,16 @@ if TYPE_CHECKING:
DEFAULT_SEND_RATE = 5_000
+# The worker before pending_commit wrote this status message while it sent a chunk.
+LEGACY_COMMIT_IN_FLIGHT_MESSAGE_PREFIX = "Starting commit of part "
+
+
+class CommitInFlightError(ValueError):
+ """A worker is sending a chunk of this import, so a resume now could skip that chunk."""
+
+ def __init__(self) -> None:
+ super().__init__("A worker is sending a chunk of this import. Try again in a few minutes.")
+
def get_aws_external_id(team: "Team") -> str:
region = (settings.CLOUD_DEPLOYMENT or "dev").lower()
@@ -61,6 +72,9 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
# Exponential backoff state (used by rust worker). Mirrors columns used by the worker.
backoff_attempt = models.IntegerField(default=0)
backoff_until = models.DateTimeField(null=True, blank=True)
+ # Worker-owned marker for the chunk being sent. A value on a row no worker holds means a worker
+ # stopped mid-commit, so the chunk may not have reached the sink.
+ pending_commit = models.JSONField(null=True, blank=True)
class Meta:
db_table = "posthog_batchimport"
@@ -116,9 +130,18 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
offset, use `resume_with_inflight_part_reset` instead. Raises ValueError if
the job is not paused.
"""
- if self.status != BatchImport.Status.PAUSED:
- raise ValueError(f"Only paused imports can be resumed (status: {self.status})")
- self._flip_to_running("Resumed by admin")
+ self.resume_from_pause("Resumed by admin")
+
+ def resume_from_pause(self, status_message: str) -> None:
+ """Resume this paused import under a row lock. Raises ValueError if it is not paused, or
+ CommitInFlightError while a worker still sends a chunk."""
+ with transaction.atomic():
+ row = BatchImport.objects.select_for_update().get(pk=self.pk)
+ if row.status != BatchImport.Status.PAUSED:
+ raise ValueError(f"Only paused imports can be resumed (status: {row.status})")
+ row.raise_if_commit_in_flight()
+ row.save(update_fields=row.reset_for_resume(status_message))
+ self.refresh_from_db()
def resume_with_inflight_part_reset(self) -> str:
"""Resume a paused import whose in-flight part has a poisoned byte offset.
@@ -135,25 +158,51 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
`resume_after_pause` does. Returns the key of the reset part. Raises
ValueError if the job is not paused or has no unfinished part.
"""
- if self.status != BatchImport.Status.PAUSED:
- raise ValueError(f"Only paused imports can be reset and resumed (status: {self.status})")
-
- _done, _total, inflight = self.parts_progress()
- if inflight is None:
- raise ValueError("No unfinished part to reset - the job has no resumable work")
-
- inflight["current_offset"] = 0
- # A stale total from a previous download would be wrong too: each download of
- # a nondeterministic export can have a different decompressed size.
- inflight["total_size"] = None
-
- # Redacted: url_list part keys are full source URLs whose query string /
- # userinfo can carry credentials, and status_message is broadly visible.
- redacted_key = str(redact_part_key(inflight.get("key")))
- self._flip_to_running(f"Resumed by admin with part {redacted_key} reset to offset 0")
+ with transaction.atomic():
+ row = BatchImport.objects.select_for_update().get(pk=self.pk)
+ if row.status != BatchImport.Status.PAUSED:
+ raise ValueError(f"Only paused imports can be reset and resumed (status: {row.status})")
+ row.raise_if_commit_in_flight()
+
+ _done, _total, inflight = row.parts_progress()
+ if inflight is None:
+ raise ValueError("No unfinished part to reset - the job has no resumable work")
+
+ inflight["current_offset"] = 0
+ # A stale total from a previous download would be wrong too: each download of
+ # a nondeterministic export can have a different decompressed size.
+ inflight["total_size"] = None
+
+ # Redacted: url_list part keys are full source URLs whose query string /
+ # userinfo can carry credentials, and status_message is broadly visible.
+ redacted_key = str(redact_part_key(inflight.get("key")))
+ row.save(
+ update_fields=[
+ *row.reset_for_resume(f"Resumed by admin with part {redacted_key} reset to offset 0"),
+ "state",
+ ]
+ )
+ self.refresh_from_db()
return redacted_key
- def _flip_to_running(self, status_message: str) -> None:
+ def commit_in_flight(self) -> bool:
+ """True while a worker holds a live lease and is between saving a chunk's offset and the
+ sink accepting it, the window in which clearing its lease can lose that chunk."""
+ if self.leased_until is None or self.leased_until <= timezone.now():
+ return False
+ return self.pending_commit is not None or (self.status_message or "").startswith(
+ LEGACY_COMMIT_IN_FLIGHT_MESSAGE_PREFIX
+ )
+
+ def raise_if_commit_in_flight(self) -> None:
+ if self.commit_in_flight():
+ raise CommitInFlightError()
+
+ def reset_for_resume(self, status_message: str) -> list[str]:
+ """Set the fields every resume path resets, and return their names for `save(update_fields=...)`.
+
+ Clears the lease so a worker can claim the row at once, and `pending_commit` because a resume
+ accepts the saved offsets."""
self.status = BatchImport.Status.RUNNING
self.status_message = status_message
self.display_status_message = None
@@ -161,19 +210,18 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
self.leased_until = None
self.backoff_attempt = 0
self.backoff_until = None
- self.save(
- update_fields=[
- "state",
- "status",
- "status_message",
- "display_status_message",
- "lease_id",
- "leased_until",
- "backoff_attempt",
- "backoff_until",
- "updated_at",
- ]
- )
+ self.pending_commit = None
+ return [
+ "status",
+ "status_message",
+ "display_status_message",
+ "lease_id",
+ "leased_until",
+ "backoff_attempt",
+ "backoff_until",
+ "pending_commit",
+ "updated_at",
+ ]
# Mostly used for manual job creation