Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): return 400 when stored source config cannot be read (#118307)

posthog[bot]224c137339a5
diff --git a/products/warehouse_sources/backend/facade/contracts.py b/products/warehouse_sources/backend/facade/contracts.py
index 3e9c83a2ed16..52aacaa70215 100644
--- a/products/warehouse_sources/backend/facade/contracts.py
+++ b/products/warehouse_sources/backend/facade/contracts.py
@@ -27,6 +27,24 @@ class UnsupportedSyncTypeError(ValueError):
"""The stored sync type is not a mode PostHog can run."""
+class UndecryptedConfigError(ValueError):
+ """Raised when a config value reaches conversion still Fernet-encrypted.
+
+ Upstream decryption of the stored job inputs failed (e.g. the key that wrote them is no
+ longer in the keychain), so the raw token would otherwise crash the field converter with an
+ opaque, secret-leaking error such as ``invalid literal for int() with base 10: 'gAAAAA...'``.
+ """
+
+
+class ConfigValueError(ValueError):
+ """Raised when a config field's value can't be converted to its declared type.
+
+ A converter (e.g. ``int``) failing on user-supplied input would otherwise crash with an
+ opaque, value-leaking builtin error such as ``invalid literal for int() with base 10: 'lakjsd'``.
+ We name the field but never its value, so nothing a user typed reaches logs or error tracking.
+ """
+
+
@dataclass(frozen=True)
class RevenueViewSyncInput:
team_id: int
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 79349256a90b..344f00a308bb 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -42,6 +42,7 @@ from products.data_warehouse.backend.facade.api import (
unpause_external_data_schedule,
update_external_job_status,
)
+from products.warehouse_sources.backend.facade.contracts import ConfigValueError, UndecryptedConfigError
from products.warehouse_sources.backend.facade.models import (
CDC_SNAPSHOT_LANE_KEY,
MAX_FULL_REFRESH_INTERVAL_DAYS,
@@ -227,6 +228,12 @@ NO_SYNC_FREQUENCY_ERROR = (
"This table has no sync frequency, so its sync cannot be scheduled. Set a sync frequency first."
)
+# The stored source settings cannot be read, for example after the instance encryption keys change.
+# Only new settings from the user can fix this, so it is not a bug to report.
+UNREADABLE_SOURCE_CONFIG_ERROR = (
+ "PostHog could not read the saved settings for this source. Reconnect the source, then try again."
+)
+
# A CDC table's own schedule loads its captured changes, and the change buffer expires them after 14
# days, so a slower schedule would lose changes for good.
CDC_MAX_SYNC_FREQUENCY_INTERVAL = dt.timedelta(days=7)
@@ -1709,7 +1716,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
if not isinstance(source_impl, WebhookSource):
return
- config = source_impl.parse_config(source.job_inputs)
+ try:
+ config = source_impl.parse_config(source.job_inputs)
+ except (UndecryptedConfigError, ConfigValueError) as e:
+ raise ValidationError(UNREADABLE_SOURCE_CONFIG_ERROR) from e
# Source pin only: webhook-sync schemas cannot carry a schema-level override (validate()
# rejects it), so there is no per-schema precedence to honor here.
effective_api_version = source_impl.resolve_api_version(source.api_version)
@@ -2369,7 +2379,10 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
source_type_enum = ExternalDataSourceType(source.source_type)
new_source = SourceRegistry.get_source(source_type_enum)
- config = new_source.parse_config(source.job_inputs)
+ try:
+ config = new_source.parse_config(source.job_inputs)
+ except (UndecryptedConfigError, ConfigValueError):
+ return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": UNREADABLE_SOURCE_CONFIG_ERROR})
effective_api_version = new_source.resolve_api_version(instance.api_version or source.api_version)
try:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
index e3b264edf264..99db95aebe8d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
@@ -8,6 +8,8 @@ import functools
import importlib
import dataclasses
+from products.warehouse_sources.backend.facade.contracts import ConfigValueError, UndecryptedConfigError
+
META_KEY = "_SOURCE_CONFIG_META"
_T = typing.TypeVar("_T")
@@ -127,24 +129,6 @@ def _noop_convert(x: typing.Any) -> typing.Any:
_ENCRYPTED_SECRET_PREFIX = "gAAAAA"
-class UndecryptedConfigError(ValueError):
- """Raised when a config value reaches conversion still Fernet-encrypted.
-
- Upstream decryption of the stored job inputs failed (e.g. the key that wrote them is no
- longer in the keychain), so the raw token would otherwise crash the field converter with an
- opaque, secret-leaking error such as ``invalid literal for int() with base 10: 'gAAAAA...'``.
- """
-
-
-class ConfigValueError(ValueError):
- """Raised when a config field's value can't be converted to its declared type.
-
- A converter (e.g. ``int``) failing on user-supplied input would otherwise crash with an
- opaque, value-leaking builtin error such as ``invalid literal for int() with base 10: 'lakjsd'``.
- We name the field but never its value, so nothing a user typed reaches logs or error tracking.
- """
-
-
def _convert_value(
convert: typing.Callable[[typing.Any], typing.Any], value: typing.Any, field_name: str
) -> typing.Any:
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 52a49ae72f59..03f3c842952b 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -42,7 +42,10 @@ from products.warehouse_sources.backend.facade.models import (
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
-from products.warehouse_sources.backend.presentation.views.external_data_schema import schema_display_status
+from products.warehouse_sources.backend.presentation.views.external_data_schema import (
+ UNREADABLE_SOURCE_CONFIG_ERROR,
+ schema_display_status,
+)
from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
INVALID_CREDENTIALS_FALLBACK_MESSAGE,
)
@@ -309,6 +312,28 @@ class TestExternalDataSchema(APIBaseTest):
assert response.status_code == 400
+ def test_incremental_fields_returns_400_when_stored_config_is_still_encrypted(self):
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_type=ExternalDataSourceType.STRIPE,
+ job_inputs={"auth_method": {"selection": "api_key", "stripe_secret_key": "gAAAAAfake-undecrypted-token"}},
+ )
+ schema = ExternalDataSchema.objects.create(
+ name="BalanceTransaction",
+ team=self.team,
+ source=source,
+ should_sync=True,
+ status=ExternalDataSchema.Status.COMPLETED,
+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,
+ )
+
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/incremental_fields",
+ )
+
+ assert response.status_code == 400
+ assert response.json() == {"message": UNREADABLE_SOURCE_CONFIG_ERROR}
+
def test_incremental_fields_missing_table_name(self):
source = ExternalDataSource.objects.create(
team=self.team,