Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): reject incremental sync without an incremental field (#119409)

posthog[bot]ce7d4796b949
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 0d980c726de4..bdd59384f459 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -1150,6 +1150,28 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
if column_names and unknown_keys:
raise ValidationError(f"'{instance.name}' has no column named {', '.join(unknown_keys)} to merge on.")
+ # An incremental or append sync saves its progress as the latest value of the incremental
+ # field. Without a field no run saves progress, so every run reads the full history again.
+ # Only a request that switches the sync type, clears the field, or turns the sync on is judged,
+ # so a table already in that state still takes other edits, such as a disable. Turning the sync
+ # on is judged because source creation saves a disabled table without a field.
+ if resulting_sync_type in (ExternalDataSchema.SyncType.INCREMENTAL, ExternalDataSchema.SyncType.APPEND):
+ stored_incremental_field = (instance.sync_type_config or {}).get("incremental_field")
+ resulting_incremental_field = (
+ data.get("incremental_field") if "incremental_field" in data else stored_incremental_field
+ )
+ switches_sync_type = resulting_sync_type != instance.sync_type
+ turns_sync_on = validated_data.get("should_sync") is True and not instance.should_sync
+ had_incremental_field = bool(stored_incremental_field)
+ if not resulting_incremental_field and (switches_sync_type or turns_sync_on or had_incremental_field):
+ replication = (
+ "incremental" if resulting_sync_type == ExternalDataSchema.SyncType.INCREMENTAL else "append only"
+ )
+ raise ValidationError(
+ f"'{instance.name}' needs an incremental field for {replication} replication. "
+ "Choose the field that tracks when rows change, or switch it to full table replication."
+ )
+
trigger_refresh = False
# Update the validated_data with incremental fields
if resulting_sync_type in incremental_style_types:
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 3af60874356b..c4a84ae7921d 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -801,6 +801,11 @@ class TestExternalDataSchema(APIBaseTest):
sync_type=initial_sync_type,
sync_type_config={
**({"primary_key_columns": persisted_keys} if persisted_keys else {}),
+ **(
+ {"incremental_field": "created_at", "incremental_field_type": "datetime"}
+ if initial_sync_type == "incremental"
+ else {}
+ ),
"schema_metadata": {"columns": columns},
},
)
@@ -1010,6 +1015,85 @@ class TestExternalDataSchema(APIBaseTest):
schema.refresh_from_db()
assert schema.sync_type_config.get("primary_key_columns") == ["id"]
+ @parameterized.expand(
+ [
+ ("switch_to_incremental_without_field", "full_refresh", None, True, {"sync_type": "incremental"}, 400),
+ (
+ "switch_to_append_with_null_field",
+ "full_refresh",
+ None,
+ True,
+ {"sync_type": "append", "incremental_field": None},
+ 400,
+ ),
+ ("clear_field_on_incremental", "incremental", "created", True, {"incremental_field": None}, 400),
+ ("enable_schema_without_field", "incremental", None, False, {"should_sync": True}, 400),
+ (
+ "unrelated_edit_on_schema_without_field",
+ "incremental",
+ None,
+ True,
+ {"sync_type": "incremental", "incremental_field": None, "should_sync": False},
+ 200,
+ ),
+ (
+ "switch_to_incremental_with_field",
+ "full_refresh",
+ None,
+ True,
+ {"sync_type": "incremental", "incremental_field": "created", "incremental_field_type": "integer"},
+ 200,
+ ),
+ ]
+ )
+ def test_update_schema_requires_incremental_field_for_cursor_sync_types(
+ self,
+ _name: str,
+ stored_sync_type: str,
+ stored_field: str | None,
+ stored_should_sync: bool,
+ request_data: dict[str, Any],
+ expected_status: int,
+ ):
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_type=ExternalDataSourceType.STRIPE,
+ job_inputs={"auth_method": {"selection": "api_key", "stripe_secret_key": "123"}},
+ )
+ schema = ExternalDataSchema.objects.create(
+ name="BalanceTransaction",
+ team=self.team,
+ source=source,
+ should_sync=stored_should_sync,
+ status=ExternalDataSchema.Status.COMPLETED,
+ sync_type=stored_sync_type,
+ sync_type_config={"incremental_field": stored_field, "incremental_field_type": "integer"}
+ if stored_field
+ else {},
+ )
+
+ with (
+ mock.patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_schema.trigger_external_data_workflow"
+ ),
+ mock.patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_schema.external_data_workflow_exists",
+ return_value=False,
+ ),
+ ):
+ response = self.client.patch(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}",
+ data=request_data,
+ )
+
+ assert response.status_code == expected_status, response.json()
+ schema.refresh_from_db()
+ if expected_status == 400:
+ assert "needs an incremental field" in response.json()["detail"]
+ assert schema.should_sync == stored_should_sync
+ assert schema.sync_type == stored_sync_type
+ assert schema.sync_type_config.get("incremental_field") == stored_field
+
def test_incremental_field_lookback_seconds_survives_reset(self):
source = ExternalDataSource.objects.create(
team=self.team,