:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(data-warehouse): read source app keys via credentials service (#120964)
Tom Owers205c67ebc55f
diff --git a/.agents/skills/implementing-warehouse-sources/SKILL.md b/.agents/skills/implementing-warehouse-sources/SKILL.mdindex bbf9b771b6e9..b38fe79b8278 100644--- a/.agents/skills/implementing-warehouse-sources/SKILL.md+++ b/.agents/skills/implementing-warehouse-sources/SKILL.md@@ -767,6 +767,10 @@ If new:- Add an `elif kind == "your-source": return OauthConfig(...)` branch in `oauth_config_for_kind()`, also in `oauth.py`.Raise `NotImplementedError("<Source> app not configured")` when the env vars are empty — that's thefail-closed message, so code and charts can ship before the secret values exist.+ This settings and `oauth_config_for_kind()` wiring is still required for the OAuth connect flow in the web app.+ Source code that runs during a sync must read PostHog-owned credentials (client ids, client secrets, developer tokens,+ service-account keys) with `credentials_service.get_secret(s)` from `sources/common/credentials_service.py`, not `settings.X`.+ Resolve them inside the function that needs them, never at import time.- If the provider's token response has **no account identifier** (e.g. Resend), decode theaccess-token JWT and set `id_path` / `name_path` from a claim (`sub`), mirroring the reddit/bingbranches in `integration_from_oauth_response`.diff --git a/products/warehouse_sources/backend/management/commands/generate_source_configs.py b/products/warehouse_sources/backend/management/commands/generate_source_configs.pyindex 5b2f48fd2f8d..8eb64ef9241c 100644--- a/products/warehouse_sources/backend/management/commands/generate_source_configs.py+++ b/products/warehouse_sources/backend/management/commands/generate_source_configs.py@@ -38,8 +38,6 @@ logger.setLevel(logging.INFO)# Leftover TODO:# - allow custom converter options (see meta ads and google ads source)-# - define metadata to add fields onto the config that come from settings, e.g:-# field: str = config.value(default_factory=config.default_from_settings("ENV_VAR"))# - Allow types to be given in `SourceFieldFileUploadJsonFormatConfig`diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/README.md b/products/warehouse_sources/backend/temporal/data_imports/sources/README.mdindex 88dbaba75334..100d494ea2d5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/README.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/README.md@@ -198,6 +198,19 @@ If your source uses OAuth (SourceFieldOauthConfig):)```+ The settings and `oauth_config_for_kind()` entries are still required for the OAuth connect flow, which runs in the web app.+ Source code that runs during a sync must not read `settings.YOUR_SOURCE_CLIENT_ID` directly.+ It reads PostHog-owned credentials through the credentials service instead:++ ```python+ from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service++ secrets = credentials_service.get_secrets(["YOUR_SOURCE_CLIENT_ID", "YOUR_SOURCE_CLIENT_SECRET"])+ ```++ `get_secret(key)` and `get_secrets(keys)` fall back to the setting of the same name when the service does not hold the key yet.+ They return an empty string when the key is configured nowhere. Resolve the keys inside the function that uses them, never at import time.+3. **Redirect URI**: Configure in external service:```textdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.pyindex 6c1998e6dd77..0d38e0c5fe5c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py@@ -6,10 +6,9 @@ from dataclasses import dataclassimport structlogfrom dateutil.relativedelta import relativedelta-from posthog.settings import integrations-from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConventionfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import initial_datetime+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_servicefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (@@ -88,20 +87,25 @@ def bing_ads_source(# Define generator function for lazy evaluation - dlt will call this when ready to fetch datadef get_rows() -> collections.abc.Iterator[list[dict]]:- developer_token = integrations.BING_ADS_DEVELOPER_TOKEN+ secrets = credentials_service.get_secrets(+ ["BING_ADS_DEVELOPER_TOKEN", "BING_ADS_CLIENT_ID", "BING_ADS_CLIENT_SECRET"]+ )+ developer_token = secrets["BING_ADS_DEVELOPER_TOKEN"]if not developer_token:raise ValueError("Bing Ads developer token not configured")# Without these the SDK posts a token request omitting client_id, and Microsoft replies with the# opaque AADSTS900144 ("request body must contain client_id") — fail fast so it isn't mis-surfaced# as a customer "reconnect your integration" error when it's really a missing PostHog config.- if not integrations.BING_ADS_CLIENT_ID or not integrations.BING_ADS_CLIENT_SECRET:+ if not secrets["BING_ADS_CLIENT_ID"] or not secrets["BING_ADS_CLIENT_SECRET"]:raise ValueError("Bing Ads OAuth application credentials not configured")client = BingAdsClient(access_token=access_token,refresh_token=refresh_token,developer_token=developer_token,+ client_id=secrets["BING_ADS_CLIENT_ID"],+ client_secret=secrets["BING_ADS_CLIENT_SECRET"],)resource = BingAdsResource(resource_name)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.pyindex 235c944e8e62..cf66437b2665 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py@@ -8,8 +8,6 @@ from bingads import AuthorizationData, OAuthTokens, OAuthWebAuthCodeGrant, Servifrom bingads.v13 import reportingfrom suds import WebFault-from posthog.settings import integrations-from products.warehouse_sources.backend.temporal.data_imports.sources.common.integration_accounts import (IntegrationAccount,)@@ -90,14 +88,14 @@ def _wrap_with_fault_detail(e: Exception, context: str) -> ValueError:class BingAdsClient:- def __init__(self, access_token: str, refresh_token: str, developer_token: str):+ def __init__(self, access_token: str, refresh_token: str, developer_token: str, client_id: str, client_secret: str):self.developer_token = developer_tokenself._customer_id: int | None = None# The SDK requires OAuth setupself.oauth = OAuthWebAuthCodeGrant(- client_id=integrations.BING_ADS_CLIENT_ID,- client_secret=integrations.BING_ADS_CLIENT_SECRET,+ client_id=client_id,+ client_secret=client_secret,redirection_uri="",)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.pyindex ce46807714ae..59d9520068e3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py@@ -290,7 +290,10 @@ class BingAdsSource(ResumableSource[BingAdsSourceConfig, BingAdsResumeConfig], O"The linked Bing Ads integration could not be found. Please reconnect your Bing Ads integration.") from e- developer_token = credentials_service.get_secret("BING_ADS_DEVELOPER_TOKEN")+ secrets = credentials_service.get_secrets(+ ["BING_ADS_DEVELOPER_TOKEN", "BING_ADS_CLIENT_ID", "BING_ADS_CLIENT_SECRET"]+ )+ developer_token = secrets["BING_ADS_DEVELOPER_TOKEN"]if not developer_token:raise ValueError("Bing Ads developer token not configured")if not integration.access_token:@@ -306,6 +309,8 @@ class BingAdsSource(ResumableSource[BingAdsSourceConfig, BingAdsResumeConfig], Oaccess_token=integration.access_token,refresh_token=integration.refresh_token,developer_token=developer_token,+ client_id=secrets["BING_ADS_CLIENT_ID"],+ client_secret=secrets["BING_ADS_CLIENT_SECRET"],)try:return client.list_accounts()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.pyindex 3254f0ec45c8..c5cb0a52e98e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py@@ -259,9 +259,15 @@ class TestBingAdsSource:return SourceDriver(BingAdsSource(), config).run(schema_name, [], **inputs)@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_campaigns(self, mock_integrations, mock_client_class):- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_campaigns(self, mock_secrets, mock_client_class):+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }mock_client = Mock()mock_client.get_data_by_resource.return_value = iter([[{"Id": "123", "Name": "Test Campaign"}]])@@ -290,9 +296,15 @@ class TestBingAdsSource:"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks")@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_report_full_refresh(self, mock_integrations, mock_client_class, mock_fetch_chunks):- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_report_full_refresh(self, mock_secrets, mock_client_class, mock_fetch_chunks):+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }mock_client = Mock()mock_client_class.return_value = mock_client@@ -332,11 +344,17 @@ class TestBingAdsSource:"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks")@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )def test_bing_ads_source_report_incremental(- self, _name, last_value, mock_integrations, mock_client_class, mock_fetch_chunks+ self, _name, last_value, mock_secrets, mock_client_class, mock_fetch_chunks):- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }mock_client = Mock()mock_client_class.return_value = mock_client@@ -371,13 +389,19 @@ class TestBingAdsSource:"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks")@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )def test_bing_ads_source_first_sync_caps_lookback_to_retention(- self, mock_integrations, mock_client_class, mock_fetch_chunks+ self, mock_secrets, mock_client_class, mock_fetch_chunks):# On the first sync (no prior incremental value) the lookback is capped to Bing's 36-month# retention window rather than reaching years back for data Bing no longer retains.- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }mock_client_class.return_value = Mock()mock_fetch_chunks.return_value = iter([[{"CampaignId": "123"}]])@@ -398,9 +422,15 @@ class TestBingAdsSource:assert mock_fetch_chunks.call_args.kwargs["start_date"] == dt.date.today() - BING_ADS_REPORT_RETENTION- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_missing_developer_token(self, mock_integrations):- mock_integrations.BING_ADS_DEVELOPER_TOKEN = None+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_missing_developer_token(self, mock_secrets):+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }result = self._drive("campaigns")@@ -416,16 +446,20 @@ class TestBingAdsSource:("both_missing", "", ""),])- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_missing_oauth_app_credentials(self, _name, client_id, client_secret, mock_integrations):+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_missing_oauth_app_credentials(self, _name, client_id, client_secret, mock_secrets):"""An empty OAuth app client id/secret raises a deterministic error instead of a doomed token request.Without the guard the SDK posts a token request omitting client_id and Microsoft returns the opaqueAADSTS900144, which was being mis-surfaced as a customer "reconnect your integration" error."""- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"- mock_integrations.BING_ADS_CLIENT_ID = client_id- mock_integrations.BING_ADS_CLIENT_SECRET = client_secret+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": client_id,+ "BING_ADS_CLIENT_SECRET": client_secret,+ }result = self._drive("campaigns")@@ -441,12 +475,16 @@ class TestBingAdsSource:("empty", ""),])- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_non_numeric_account_id(self, _name, account_id, mock_integrations):+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_non_numeric_account_id(self, _name, account_id, mock_secrets):"""A non-numeric account ID raises a deterministic, non-retryable error instead of a bare int() crash."""- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"- mock_integrations.BING_ADS_CLIENT_ID = "test_client_id"- mock_integrations.BING_ADS_CLIENT_SECRET = "test_client_secret"+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }result = self._drive("campaigns", account_id=account_id)@@ -455,11 +493,15 @@ class TestBingAdsSource:assert "Bing Ads Account ID must be numeric" in str(result.raised)assert result.requests == []- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")- def test_bing_ads_source_incremental_missing_field(self, mock_integrations):- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"- mock_integrations.BING_ADS_CLIENT_ID = "test_client_id"- mock_integrations.BING_ADS_CLIENT_SECRET = "test_client_secret"+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"+ )+ def test_bing_ads_source_incremental_missing_field(self, mock_secrets):+ mock_secrets.return_value = {+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ }result = self._drive("campaign_performance_report",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.pyindex 284a1fefb024..12a67f4b6953 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py@@ -81,6 +81,8 @@ class TestBingAdsClient:self.access_token = "test_access_token"self.refresh_token = "test_refresh_token"self.developer_token = "test_developer_token"+ self.client_id = "test_client_id"+ self.client_secret = "test_client_secret"self.account_id = 12345self.customer_id = 67890@@ -105,7 +107,9 @@ class TestBingAdsClient:instance.GetAccountsInfo.side_effect = accounts_for- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )result = client.list_accounts()assert result == [@@ -136,7 +140,9 @@ class TestBingAdsClient:instance.GetUser.return_value = mock.MagicMock(User=mock.MagicMock(CustomerId=111))instance.GetCustomersInfo.side_effect = _make_webfault("Server raised fault: 'Invalid client data.'", None)- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError, match="Failed to list Bing Ads accounts"):client.list_accounts()@@ -155,7 +161,9 @@ class TestBingAdsClient:"invalid_grant The provided authorization grant has expired or been revoked.")- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:client.get_customer_id()@@ -174,7 +182,9 @@ class TestBingAdsClient:"HTTPSConnectionPool(host='bingads.microsoft.com', port=443): Max retries exceeded")- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:client.get_customer_id()@@ -197,7 +207,9 @@ class TestBingAdsClient:mock_client_instance = mock_service_client.return_valuemock_client_instance.GetUser.side_effect = webfault- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:client.get_customer_id()@@ -217,7 +229,9 @@ class TestBingAdsClient:mock_client_instance = mock_service_client.return_valuemock_client_instance.GetUser.side_effect = webfault- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:client.get_customer_id()@@ -256,7 +270,9 @@ class TestBingAdsClient:mock_reporting.ReportingServiceManager.side_effect = fake_managermock_iter_pages.return_value = iter([])- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )# The report is a generator now, so the manager is only built once it is drained.list(client.get_performance_report(@@ -294,7 +310,9 @@ class TestBingAdsClient:mock_client_instance = mock_service_client.return_valuemock_client_instance.GetCampaignsByAccountId.return_value = mock_campaigns- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )result = list(client.get_campaigns(self.account_id, self.customer_id))assert len(result) == 1@@ -323,7 +341,9 @@ class TestBingAdsClient:mock_client_instance = mock_service_client.return_valuemock_client_instance.GetCampaignsByAccountId.side_effect = webfault- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:list(client.get_campaigns(self.account_id, self.customer_id))@@ -344,7 +364,9 @@ class TestBingAdsClient:"HTTPSConnectionPool(host='bingads.microsoft.com', port=443): Max retries exceeded")- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)+ client = BingAdsClient(+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret+ )with pytest.raises(ValueError) as exc_info:list(client.get_campaigns(self.account_id, self.customer_id))diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.pyindex a4a24b6a7363..5a6449f26159 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py@@ -250,8 +250,12 @@ class TestBingAdsSource:],)@mock.patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.credentials_service.get_secret",- return_value="dev-token",+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.credentials_service.get_secrets",+ return_value={+ "BING_ADS_DEVELOPER_TOKEN": "dev-token",+ "BING_ADS_CLIENT_ID": "test_client_id",+ "BING_ADS_CLIENT_SECRET": "test_client_secret",+ },)@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.BingAdsClient")@mock.patch.object(BingAdsSource, "get_oauth_integration")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.pyindex eb78a0a43eb6..e3b264edf264 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py@@ -829,34 +829,3 @@ def str_to_optional_list(s: str | list[typing.Any] | None) -> list[str] | None:raise TypeError(f"expected a string, list, or None, got {type(s).__name__}")values = [value for value in values if value]return values or None---_DefaultType = typing.TypeVar("_DefaultType")---def default_from_settings(- key: str, converter: typing.Callable[[str], _DefaultType] = _noop_convert-) -> typing.Callable[[], _DefaultType]:- """Return a default factory that obtains the value from app settings."""-- def default_factory() -> _DefaultType:- from django.conf import settings-- return converter(getattr(settings, key))-- return default_factory---def default_from_env(- key: str, converter: typing.Callable[[str], _DefaultType] = _noop_convert-) -> typing.Callable[[], _DefaultType]:- """Return a default factory that obtains the value from an env variable."""-- def default_factory() -> _DefaultType:- import os-- value = os.environ[key]-- return converter(value)-- return default_factorydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.pyindex 67359dad3b7b..e906e0e1a22e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py@@ -81,17 +81,5 @@ class GoogleAdsServiceAccountSourceConfig(config.Config):customer_id: str = config.value(converter=clean_customer_id)- private_key: str = config.value(- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY")- )- private_key_id: str = config.value(- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID")- )- client_email: str = config.value(- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL")- )- token_uri: str = config.value(default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI"))- developer_token: str = config.value(default_factory=config.default_from_settings("GOOGLE_ADS_DEVELOPER_TOKEN"))-GoogleAdsSourceConfigUnion = GoogleAdsServiceAccountSourceConfig | GoogleAdsSourceConfigdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.pyindex 21e58ce302c5..973c972b7c95 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py@@ -256,16 +256,25 @@ def google_ads_client(config: GoogleAdsSourceConfigUnion, team_id: int) -> Googlclient = _load_client_with_transient_retry(config_dict)else:+ resolved = credentials_service.get_secrets(+ [+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY",+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID",+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL",+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI",+ "GOOGLE_ADS_DEVELOPER_TOKEN",+ ]+ )credentials = service_account.Credentials.from_service_account_info({- "private_key": config.private_key,- "private_key_id": config.private_key_id,- "token_uri": config.token_uri,- "client_email": config.client_email,+ "private_key": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY"],+ "private_key_id": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID"],+ "token_uri": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI"],+ "client_email": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL"],},scopes=["https://www.googleapis.com/auth/adwords"],)- client = GoogleAdsClient(credentials=credentials, developer_token=config.developer_token)+ client = GoogleAdsClient(credentials=credentials, developer_token=resolved["GOOGLE_ADS_DEVELOPER_TOKEN"])return clientdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.pyindex 6b7ddac4ebe6..4a5477756423 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py@@ -1,11 +1,12 @@+from datetime import timedeltafrom typing import Optional, cast-from django.conf import settings-import gspreadimport structlogfrom google.auth import exceptions as google_auth_exceptions+from posthog.cache_utils import cache_for+from products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (DataWarehouseSourceCategory,@@ -15,6 +16,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfigType,)from products.warehouse_sources.backend.models.external_data_schema import SCHEMA_RESOURCE_ID_METADATA_KEY+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_servicefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (UNVERSIONED_API_VERSION,FieldType,@@ -37,6 +39,12 @@ from products.warehouse_sources.backend.types import ExternalDataSourceTypelogger = structlog.get_logger(__name__)+# The source config is built on many web requests, and this address is display text, not a secret.+@cache_for(timedelta(minutes=5))+def _service_account_email() -> str:+ return credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")++@SourceRegistry.registerclass GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):codebase_detection = detect(@@ -187,16 +195,18 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):except gspread.SpreadsheetNotFound:# The Sheets API answers an unshared sheet with a 404, the same as a deleted one, so# sharing is at least as likely as a wrong URL. Mirror the sync-time message.+ service_account_email = credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")return (False,"PostHog couldn't find a sheet at that URL. Check the URL, and share the sheet with our service "- f"account ({settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}) as a Viewer.",+ f"account ({service_account_email}) as a Viewer.",)except PermissionError:+ service_account_email = credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")return (False,"PostHog does not have access to this spreadsheet. Share it with our service account "- f"({settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}) as a Viewer, then try again. "+ f"({service_account_email}) as a Viewer, then try again. ""See https://posthog.com/docs/cdp/sources/google-sheets for more.",)except gspread.exceptions.APIError as e:@@ -237,6 +247,7 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):@propertydef get_source_config(self) -> SourceConfig:+ service_account_email = _service_account_email()return SourceConfig(name=ExternalDataSourceType.GOOGLESHEETS,category=DataWarehouseSourceCategory.PRODUCTIVITY,@@ -255,7 +266,7 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):type=SourceFieldInputConfigType.TEXT,required=True,placeholder="",- caption=f'Share the sheet with our service account by entering **{settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}** into the "Add people" field. We only require "Viewer" permissions to sync the sheet.',+ caption=f'Share the sheet with our service account by entering **{service_account_email}** into the "Add people" field. We only require "Viewer" permissions to sync the sheet.',secret=False,)],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.pyindex cb4e079d7761..d032483343da 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py@@ -4,14 +4,13 @@ from typing import Anyfrom urllib.parse import quotefrom zoneinfo import ZoneInfo-from django.conf import settings-import requestsfrom requests_oauthlib import OAuth1from posthog.dataclasses import frozenfrom posthog.models.integration.model import Integration+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_servicefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -95,14 +94,13 @@ class TwitterAdsClient:secret = integration.sensitive_config.get("oauth_token_secret")if not token or not secret:raise ValueError(MISSING_INTEGRATION)- if not settings.TWITTER_ADS_CONSUMER_KEY or not settings.TWITTER_ADS_CONSUMER_SECRET:+ consumer = credentials_service.get_secrets(["TWITTER_ADS_CONSUMER_KEY", "TWITTER_ADS_CONSUMER_SECRET"])+ consumer_key = consumer["TWITTER_ADS_CONSUMER_KEY"]+ consumer_secret = consumer["TWITTER_ADS_CONSUMER_SECRET"]+ if not consumer_key or not consumer_secret:raise ValueError(MISSING_APP)- self.session = make_tracked_session(- allow_redirects=False, redact_values=(token, secret, settings.TWITTER_ADS_CONSUMER_SECRET)- )- self.session.auth = OAuth1(- settings.TWITTER_ADS_CONSUMER_KEY, settings.TWITTER_ADS_CONSUMER_SECRET, token, secret- )+ self.session = make_tracked_session(allow_redirects=False, redact_values=(token, secret, consumer_secret))+ self.session.auth = OAuth1(consumer_key, consumer_secret, token, secret)self.base_url = f"https://ads-api.x.com/{api_version}"def get(self, path: str, params: dict[str, str | int] | None = None) -> dict[str, Any]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.pyindex 198a4a78ac31..285678eb86d5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py@@ -6,8 +6,6 @@ from dataclasses import dataclassimport pytestfrom unittest import mock-from django.test import override_settings-import pyarrow as pafrom posthog.models import Team@@ -62,78 +60,40 @@ def developer_token() -> str:@pytest.fixture-def service_account_config() -> dict[str, str]:- """Return a Service Account configuration dictionary to use in tests."""+def service_account_secrets(developer_token: str):+ """Serve the platform service account credentials through the credentials service wrapper."""credentials_file_path = os.environ["GOOGLE_SERVICE_ACCOUNT_CREDENTIALS"]with open(credentials_file_path) as f:credentials = json.load(f)- return {- "private_key": credentials["private_key"],- "private_key_id": credentials["private_key_id"],- "token_uri": credentials["token_uri"],- "client_email": credentials["client_email"],- }---def test_google_ads_source_config_loads(customer_id: str, developer_token: str):- private_key = "private_key"- private_key_id = "id"- client_email = "[email protected]"- token_uri = "https://posthog.com"-- job_inputs = {- "resource_name": "campaign",- "customer_id": customer_id,- }-- with override_settings(- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY=private_key,- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID=private_key_id,- GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL=client_email,- GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI=token_uri,- GOOGLE_ADS_DEVELOPER_TOKEN=developer_token,- ):- cfg = GoogleAdsServiceAccountSourceConfig.from_dict(job_inputs)-- assert cfg.private_key == private_key- assert cfg.private_key_id == private_key_id- assert cfg.client_email == client_email- assert cfg.token_uri == token_uri- assert cfg.developer_token == developer_token- assert cfg.customer_id == customer_id---def test_google_ads_source_config_handles_customer_id_with_dashes(developer_token: str):- private_key = "private_key"- private_key_id = "id"- client_email = "[email protected]"- token_uri = "https://posthog.com"-- job_inputs = {- "resource_name": "campaign",- "customer_id": "111-111-1111",- }-- with override_settings(- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY=private_key,- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID=private_key_id,- GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL=client_email,- GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI=token_uri,- GOOGLE_ADS_DEVELOPER_TOKEN=developer_token,+ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.credentials_service.get_secrets",+ return_value={+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY": credentials["private_key"],+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID": credentials["private_key_id"],+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI": credentials["token_uri"],+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL": credentials["client_email"],+ "GOOGLE_ADS_DEVELOPER_TOKEN": developer_token,+ },):- cfg = GoogleAdsServiceAccountSourceConfig.from_dict(job_inputs)+ yield+++ "raw_customer_id,expected",+ [+ ("1111111111", "1111111111"),+ ("111-111-1111", "1111111111"),+ ],+)+def test_google_ads_service_account_config_loads(raw_customer_id: str, expected: str):+ cfg = GoogleAdsServiceAccountSourceConfig.from_dict({"resource_name": "campaign", "customer_id": raw_customer_id})- assert cfg.private_key == private_key- assert cfg.private_key_id == private_key_id- assert cfg.client_email == client_email- assert cfg.token_uri == token_uri- assert cfg.developer_token == developer_token- assert cfg.customer_id == "1111111111"+ assert cfg.customer_id == expected@SKIP_IF_MISSING_GOOGLE_ADS_CREDENTIALS-def test_get_schemas(customer_id: str, developer_token: str, service_account_config: dict[str, str]):+def test_get_schemas(customer_id: str, service_account_secrets: None):"""Test get_schemas returns well-known schemas.This test is not exhaustive and merely limits itself to asserting a handful@@ -142,9 +102,7 @@ def test_get_schemas(customer_id: str, developer_token: str, service_account_conorg = Organization.objects.create(name="org")team = Team.objects.create(organization=org)- cfg = GoogleAdsServiceAccountSourceConfig(- customer_id=customer_id, developer_token=developer_token, **service_account_config- )+ cfg = GoogleAdsServiceAccountSourceConfig(customer_id=customer_id)schemas = get_schemas(cfg, team_id=team.id, api_version=GoogleAdsSource().default_version)assert "campaign" in schemas@@ -171,7 +129,7 @@ def test_get_schemas(customer_id: str, developer_token: str, service_account_con@SKIP_IF_MISSING_GOOGLE_ADS_CREDENTIALS-def test_google_ads_source(customer_id: str, developer_token: str, service_account_config: dict[str, str]):+def test_google_ads_source(customer_id: str, service_account_secrets: None):"""Test google_ads_source can iterate rows.This test is not exhaustive and merely limits itself to attempting to@@ -181,9 +139,7 @@ def test_google_ads_source(customer_id: str, developer_token: str, service_accouorg = Organization.objects.create(name="org")team = Team.objects.create(organization=org)- cfg = GoogleAdsServiceAccountSourceConfig(- customer_id=customer_id, developer_token=developer_token, **service_account_config- )+ cfg = GoogleAdsServiceAccountSourceConfig(customer_id=customer_id)for resource in ("campaign","campaign_stats",@@ -605,6 +561,9 @@ class TestGoogleAdsClientStaleConnection:call_order = [name for name, _, _ in manager.mock_calls]assert call_order.index("close_old_connections") < call_order.index("integration_get")+ @mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.credentials_service.get_secrets"+ )@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.service_account")@@ -615,20 +574,28 @@ class TestGoogleAdsClientStaleConnection:"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.close_old_connections")def test_service_account_client_does_not_touch_django_connections(- self, mock_close_old_connections, mock_client_cls, mock_service_account+ self, mock_close_old_connections, mock_client_cls, mock_service_account, mock_get_secrets):- config = GoogleAdsServiceAccountSourceConfig(- customer_id="1234567890",- private_key="pk",- private_key_id="pk_id",- client_email="[email protected]",- token_uri="https://example.com",- developer_token="dev",- )+ mock_get_secrets.return_value = {+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY": "pk",+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID": "pk_id",+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL": "[email protected]",+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI": "https://example.com",+ "GOOGLE_ADS_DEVELOPER_TOKEN": "dev",+ }+ config = GoogleAdsServiceAccountSourceConfig(customer_id="1234567890")google_ads_client(config, team_id=7)mock_close_old_connections.assert_not_called()+ mock_service_account.Credentials.from_service_account_info.assert_called_once()+ assert mock_service_account.Credentials.from_service_account_info.call_args.args[0] == {+ "private_key": "pk",+ "private_key_id": "pk_id",+ "token_uri": "https://example.com",+ "client_email": "[email protected]",+ }+ assert mock_client_cls.call_args.kwargs["developer_token"] == "dev"def test_source_registration_does_not_import_google_ads_sdk(google_ads_sdk_leak_check) -> None: