:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): return 400 when stored source config cannot be read (#118307)
posthog[bot]224c137339a5
diff --git a/products/warehouse_sources/backend/facade/contracts.py b/products/warehouse_sources/backend/facade/contracts.pyindex 3e9c83a2ed16..52aacaa70215 100644--- a/products/warehouse_sources/backend/facade/contracts.py+++ b/products/warehouse_sources/backend/facade/contracts.py@@ -27,6 +27,24 @@ class UnsupportedSyncTypeError(ValueError):"""The stored sync type is not a mode PostHog can run."""+class UndecryptedConfigError(ValueError):+ """Raised when a config value reaches conversion still Fernet-encrypted.++ Upstream decryption of the stored job inputs failed (e.g. the key that wrote them is no+ longer in the keychain), so the raw token would otherwise crash the field converter with an+ opaque, secret-leaking error such as ``invalid literal for int() with base 10: 'gAAAAA...'``.+ """+++class ConfigValueError(ValueError):+ """Raised when a config field's value can't be converted to its declared type.++ A converter (e.g. ``int``) failing on user-supplied input would otherwise crash with an+ opaque, value-leaking builtin error such as ``invalid literal for int() with base 10: 'lakjsd'``.+ We name the field but never its value, so nothing a user typed reaches logs or error tracking.+ """++@dataclass(frozen=True)class RevenueViewSyncInput:team_id: intdiff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 79349256a90b..344f00a308bb 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -42,6 +42,7 @@ from products.data_warehouse.backend.facade.api import (unpause_external_data_schedule,update_external_job_status,)+from products.warehouse_sources.backend.facade.contracts import ConfigValueError, UndecryptedConfigErrorfrom products.warehouse_sources.backend.facade.models import (CDC_SNAPSHOT_LANE_KEY,MAX_FULL_REFRESH_INTERVAL_DAYS,@@ -227,6 +228,12 @@ NO_SYNC_FREQUENCY_ERROR = ("This table has no sync frequency, so its sync cannot be scheduled. Set a sync frequency first.")+# The stored source settings cannot be read, for example after the instance encryption keys change.+# Only new settings from the user can fix this, so it is not a bug to report.+UNREADABLE_SOURCE_CONFIG_ERROR = (+ "PostHog could not read the saved settings for this source. Reconnect the source, then try again."+)+# A CDC table's own schedule loads its captured changes, and the change buffer expires them after 14# days, so a slower schedule would lose changes for good.CDC_MAX_SYNC_FREQUENCY_INTERVAL = dt.timedelta(days=7)@@ -1709,7 +1716,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersif not isinstance(source_impl, WebhookSource):return- config = source_impl.parse_config(source.job_inputs)+ try:+ config = source_impl.parse_config(source.job_inputs)+ except (UndecryptedConfigError, ConfigValueError) as e:+ raise ValidationError(UNREADABLE_SOURCE_CONFIG_ERROR) from e# Source pin only: webhook-sync schemas cannot carry a schema-level override (validate()# rejects it), so there is no per-schema precedence to honor here.effective_api_version = source_impl.resolve_api_version(source.api_version)@@ -2369,7 +2379,10 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):source_type_enum = ExternalDataSourceType(source.source_type)new_source = SourceRegistry.get_source(source_type_enum)- config = new_source.parse_config(source.job_inputs)+ try:+ config = new_source.parse_config(source.job_inputs)+ except (UndecryptedConfigError, ConfigValueError):+ return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": UNREADABLE_SOURCE_CONFIG_ERROR})effective_api_version = new_source.resolve_api_version(instance.api_version or source.api_version)try:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.pyindex e3b264edf264..99db95aebe8d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py@@ -8,6 +8,8 @@ import functoolsimport importlibimport dataclasses+from products.warehouse_sources.backend.facade.contracts import ConfigValueError, UndecryptedConfigError+META_KEY = "_SOURCE_CONFIG_META"_T = typing.TypeVar("_T")@@ -127,24 +129,6 @@ def _noop_convert(x: typing.Any) -> typing.Any:_ENCRYPTED_SECRET_PREFIX = "gAAAAA"-class UndecryptedConfigError(ValueError):- """Raised when a config value reaches conversion still Fernet-encrypted.-- Upstream decryption of the stored job inputs failed (e.g. the key that wrote them is no- longer in the keychain), so the raw token would otherwise crash the field converter with an- opaque, secret-leaking error such as ``invalid literal for int() with base 10: 'gAAAAA...'``.- """---class ConfigValueError(ValueError):- """Raised when a config field's value can't be converted to its declared type.-- A converter (e.g. ``int``) failing on user-supplied input would otherwise crash with an- opaque, value-leaking builtin error such as ``invalid literal for int() with base 10: 'lakjsd'``.- We name the field but never its value, so nothing a user typed reaches logs or error tracking.- """--def _convert_value(convert: typing.Callable[[typing.Any], typing.Any], value: typing.Any, field_name: str) -> typing.Any:diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 52a49ae72f59..03f3c842952b 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -42,7 +42,10 @@ from products.warehouse_sources.backend.facade.models import (update_sync_type_config_keys,)from products.warehouse_sources.backend.facade.types import ExternalDataSourceType-from products.warehouse_sources.backend.presentation.views.external_data_schema import schema_display_status+from products.warehouse_sources.backend.presentation.views.external_data_schema import (+ UNREADABLE_SOURCE_CONFIG_ERROR,+ schema_display_status,+)from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (INVALID_CREDENTIALS_FALLBACK_MESSAGE,)@@ -309,6 +312,28 @@ class TestExternalDataSchema(APIBaseTest):assert response.status_code == 400+ def test_incremental_fields_returns_400_when_stored_config_is_still_encrypted(self):+ source = ExternalDataSource.objects.create(+ team=self.team,+ source_type=ExternalDataSourceType.STRIPE,+ job_inputs={"auth_method": {"selection": "api_key", "stripe_secret_key": "gAAAAAfake-undecrypted-token"}},+ )+ schema = ExternalDataSchema.objects.create(+ name="BalanceTransaction",+ team=self.team,+ source=source,+ should_sync=True,+ status=ExternalDataSchema.Status.COMPLETED,+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,+ )++ response = self.client.post(+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/incremental_fields",+ )++ assert response.status_code == 400+ assert response.json() == {"message": UNREADABLE_SOURCE_CONFIG_ERROR}+def test_incremental_fields_missing_table_name(self):source = ExternalDataSource.objects.create(team=self.team,