:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): reject incremental sync without an incremental field (#119409)
posthog[bot]ce7d4796b949
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 0d980c726de4..bdd59384f459 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -1150,6 +1150,28 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersif column_names and unknown_keys:raise ValidationError(f"'{instance.name}' has no column named {', '.join(unknown_keys)} to merge on.")+ # An incremental or append sync saves its progress as the latest value of the incremental+ # field. Without a field no run saves progress, so every run reads the full history again.+ # Only a request that switches the sync type, clears the field, or turns the sync on is judged,+ # so a table already in that state still takes other edits, such as a disable. Turning the sync+ # on is judged because source creation saves a disabled table without a field.+ if resulting_sync_type in (ExternalDataSchema.SyncType.INCREMENTAL, ExternalDataSchema.SyncType.APPEND):+ stored_incremental_field = (instance.sync_type_config or {}).get("incremental_field")+ resulting_incremental_field = (+ data.get("incremental_field") if "incremental_field" in data else stored_incremental_field+ )+ switches_sync_type = resulting_sync_type != instance.sync_type+ turns_sync_on = validated_data.get("should_sync") is True and not instance.should_sync+ had_incremental_field = bool(stored_incremental_field)+ if not resulting_incremental_field and (switches_sync_type or turns_sync_on or had_incremental_field):+ replication = (+ "incremental" if resulting_sync_type == ExternalDataSchema.SyncType.INCREMENTAL else "append only"+ )+ raise ValidationError(+ f"'{instance.name}' needs an incremental field for {replication} replication. "+ "Choose the field that tracks when rows change, or switch it to full table replication."+ )+trigger_refresh = False# Update the validated_data with incremental fieldsif resulting_sync_type in incremental_style_types:diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 3af60874356b..c4a84ae7921d 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -801,6 +801,11 @@ class TestExternalDataSchema(APIBaseTest):sync_type=initial_sync_type,sync_type_config={**({"primary_key_columns": persisted_keys} if persisted_keys else {}),+ **(+ {"incremental_field": "created_at", "incremental_field_type": "datetime"}+ if initial_sync_type == "incremental"+ else {}+ ),"schema_metadata": {"columns": columns},},)@@ -1010,6 +1015,85 @@ class TestExternalDataSchema(APIBaseTest):schema.refresh_from_db()assert schema.sync_type_config.get("primary_key_columns") == ["id"]+ @parameterized.expand(+ [+ ("switch_to_incremental_without_field", "full_refresh", None, True, {"sync_type": "incremental"}, 400),+ (+ "switch_to_append_with_null_field",+ "full_refresh",+ None,+ True,+ {"sync_type": "append", "incremental_field": None},+ 400,+ ),+ ("clear_field_on_incremental", "incremental", "created", True, {"incremental_field": None}, 400),+ ("enable_schema_without_field", "incremental", None, False, {"should_sync": True}, 400),+ (+ "unrelated_edit_on_schema_without_field",+ "incremental",+ None,+ True,+ {"sync_type": "incremental", "incremental_field": None, "should_sync": False},+ 200,+ ),+ (+ "switch_to_incremental_with_field",+ "full_refresh",+ None,+ True,+ {"sync_type": "incremental", "incremental_field": "created", "incremental_field_type": "integer"},+ 200,+ ),+ ]+ )+ def test_update_schema_requires_incremental_field_for_cursor_sync_types(+ self,+ _name: str,+ stored_sync_type: str,+ stored_field: str | None,+ stored_should_sync: bool,+ request_data: dict[str, Any],+ expected_status: int,+ ):+ source = ExternalDataSource.objects.create(+ team=self.team,+ source_type=ExternalDataSourceType.STRIPE,+ job_inputs={"auth_method": {"selection": "api_key", "stripe_secret_key": "123"}},+ )+ schema = ExternalDataSchema.objects.create(+ name="BalanceTransaction",+ team=self.team,+ source=source,+ should_sync=stored_should_sync,+ status=ExternalDataSchema.Status.COMPLETED,+ sync_type=stored_sync_type,+ sync_type_config={"incremental_field": stored_field, "incremental_field_type": "integer"}+ if stored_field+ else {},+ )++ with (+ mock.patch(+ "products.warehouse_sources.backend.presentation.views.external_data_schema.trigger_external_data_workflow"+ ),+ mock.patch(+ "products.warehouse_sources.backend.presentation.views.external_data_schema.external_data_workflow_exists",+ return_value=False,+ ),+ ):+ response = self.client.patch(+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}",+ data=request_data,+ )++ assert response.status_code == expected_status, response.json()+ schema.refresh_from_db()+ if expected_status == 400:+ assert "needs an incremental field" in response.json()["detail"]+ assert schema.should_sync == stored_should_sync+ assert schema.sync_type == stored_sync_type+ assert schema.sync_type_config.get("incremental_field") == stored_field+def test_incremental_field_lookback_seconds_survives_reset(self):source = ExternalDataSource.objects.create(team=self.team,