Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(data-warehouse): read source app keys via credentials service (#120964)

Tom Owers205c67ebc55f
diff --git a/.agents/skills/implementing-warehouse-sources/SKILL.md b/.agents/skills/implementing-warehouse-sources/SKILL.md
index bbf9b771b6e9..b38fe79b8278 100644
--- a/.agents/skills/implementing-warehouse-sources/SKILL.md
+++ b/.agents/skills/implementing-warehouse-sources/SKILL.md
@@ -767,6 +767,10 @@ If new:
- Add an `elif kind == "your-source": return OauthConfig(...)` branch in `oauth_config_for_kind()`, also in `oauth.py`.
Raise `NotImplementedError("<Source> app not configured")` when the env vars are empty — that's the
fail-closed message, so code and charts can ship before the secret values exist.
+ This settings and `oauth_config_for_kind()` wiring is still required for the OAuth connect flow in the web app.
+ Source code that runs during a sync must read PostHog-owned credentials (client ids, client secrets, developer tokens,
+ service-account keys) with `credentials_service.get_secret(s)` from `sources/common/credentials_service.py`, not `settings.X`.
+ Resolve them inside the function that needs them, never at import time.
- If the provider's token response has **no account identifier** (e.g. Resend), decode the
access-token JWT and set `id_path` / `name_path` from a claim (`sub`), mirroring the reddit/bing
branches in `integration_from_oauth_response`.
diff --git a/products/warehouse_sources/backend/management/commands/generate_source_configs.py b/products/warehouse_sources/backend/management/commands/generate_source_configs.py
index 5b2f48fd2f8d..8eb64ef9241c 100644
--- a/products/warehouse_sources/backend/management/commands/generate_source_configs.py
+++ b/products/warehouse_sources/backend/management/commands/generate_source_configs.py
@@ -38,8 +38,6 @@ logger.setLevel(logging.INFO)
# Leftover TODO:
# - allow custom converter options (see meta ads and google ads source)
-# - define metadata to add fields onto the config that come from settings, e.g:
-# field: str = config.value(default_factory=config.default_from_settings("ENV_VAR"))
# - Allow types to be given in `SourceFieldFileUploadJsonFormatConfig`
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/README.md b/products/warehouse_sources/backend/temporal/data_imports/sources/README.md
index 88dbaba75334..100d494ea2d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/README.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/README.md
@@ -198,6 +198,19 @@ If your source uses OAuth (SourceFieldOauthConfig):
)
```
+ The settings and `oauth_config_for_kind()` entries are still required for the OAuth connect flow, which runs in the web app.
+ Source code that runs during a sync must not read `settings.YOUR_SOURCE_CLIENT_ID` directly.
+ It reads PostHog-owned credentials through the credentials service instead:
+
+ ```python
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
+
+ secrets = credentials_service.get_secrets(["YOUR_SOURCE_CLIENT_ID", "YOUR_SOURCE_CLIENT_SECRET"])
+ ```
+
+ `get_secret(key)` and `get_secrets(keys)` fall back to the setting of the same name when the service does not hold the key yet.
+ They return an empty string when the key is configured nowhere. Resolve the keys inside the function that uses them, never at import time.
+
3. **Redirect URI**: Configure in external service:
```text
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py
index 6c1998e6dd77..0d38e0c5fe5c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/bing_ads.py
@@ -6,10 +6,9 @@ from dataclasses import dataclass
import structlog
from dateutil.relativedelta import relativedelta
-from posthog.settings import integrations
-
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import initial_datetime
+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (
@@ -88,20 +87,25 @@ def bing_ads_source(
# Define generator function for lazy evaluation - dlt will call this when ready to fetch data
def get_rows() -> collections.abc.Iterator[list[dict]]:
- developer_token = integrations.BING_ADS_DEVELOPER_TOKEN
+ secrets = credentials_service.get_secrets(
+ ["BING_ADS_DEVELOPER_TOKEN", "BING_ADS_CLIENT_ID", "BING_ADS_CLIENT_SECRET"]
+ )
+ developer_token = secrets["BING_ADS_DEVELOPER_TOKEN"]
if not developer_token:
raise ValueError("Bing Ads developer token not configured")
# Without these the SDK posts a token request omitting client_id, and Microsoft replies with the
# opaque AADSTS900144 ("request body must contain client_id") — fail fast so it isn't mis-surfaced
# as a customer "reconnect your integration" error when it's really a missing PostHog config.
- if not integrations.BING_ADS_CLIENT_ID or not integrations.BING_ADS_CLIENT_SECRET:
+ if not secrets["BING_ADS_CLIENT_ID"] or not secrets["BING_ADS_CLIENT_SECRET"]:
raise ValueError("Bing Ads OAuth application credentials not configured")
client = BingAdsClient(
access_token=access_token,
refresh_token=refresh_token,
developer_token=developer_token,
+ client_id=secrets["BING_ADS_CLIENT_ID"],
+ client_secret=secrets["BING_ADS_CLIENT_SECRET"],
)
resource = BingAdsResource(resource_name)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py
index 235c944e8e62..cf66437b2665 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/client.py
@@ -8,8 +8,6 @@ from bingads import AuthorizationData, OAuthTokens, OAuthWebAuthCodeGrant, Servi
from bingads.v13 import reporting
from suds import WebFault
-from posthog.settings import integrations
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.integration_accounts import (
IntegrationAccount,
)
@@ -90,14 +88,14 @@ def _wrap_with_fault_detail(e: Exception, context: str) -> ValueError:
class BingAdsClient:
- def __init__(self, access_token: str, refresh_token: str, developer_token: str):
+ def __init__(self, access_token: str, refresh_token: str, developer_token: str, client_id: str, client_secret: str):
self.developer_token = developer_token
self._customer_id: int | None = None
# The SDK requires OAuth setup
self.oauth = OAuthWebAuthCodeGrant(
- client_id=integrations.BING_ADS_CLIENT_ID,
- client_secret=integrations.BING_ADS_CLIENT_SECRET,
+ client_id=client_id,
+ client_secret=client_secret,
redirection_uri="",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py
index ce46807714ae..59d9520068e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/source.py
@@ -290,7 +290,10 @@ class BingAdsSource(ResumableSource[BingAdsSourceConfig, BingAdsResumeConfig], O
"The linked Bing Ads integration could not be found. Please reconnect your Bing Ads integration."
) from e
- developer_token = credentials_service.get_secret("BING_ADS_DEVELOPER_TOKEN")
+ secrets = credentials_service.get_secrets(
+ ["BING_ADS_DEVELOPER_TOKEN", "BING_ADS_CLIENT_ID", "BING_ADS_CLIENT_SECRET"]
+ )
+ developer_token = secrets["BING_ADS_DEVELOPER_TOKEN"]
if not developer_token:
raise ValueError("Bing Ads developer token not configured")
if not integration.access_token:
@@ -306,6 +309,8 @@ class BingAdsSource(ResumableSource[BingAdsSourceConfig, BingAdsResumeConfig], O
access_token=integration.access_token,
refresh_token=integration.refresh_token,
developer_token=developer_token,
+ client_id=secrets["BING_ADS_CLIENT_ID"],
+ client_secret=secrets["BING_ADS_CLIENT_SECRET"],
)
try:
return client.list_accounts()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py
index 3254f0ec45c8..c5cb0a52e98e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py
@@ -259,9 +259,15 @@ class TestBingAdsSource:
return SourceDriver(BingAdsSource(), config).run(schema_name, [], **inputs)
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_campaigns(self, mock_integrations, mock_client_class):
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_campaigns(self, mock_secrets, mock_client_class):
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
mock_client = Mock()
mock_client.get_data_by_resource.return_value = iter([[{"Id": "123", "Name": "Test Campaign"}]])
@@ -290,9 +296,15 @@ class TestBingAdsSource:
"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks"
)
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_report_full_refresh(self, mock_integrations, mock_client_class, mock_fetch_chunks):
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_report_full_refresh(self, mock_secrets, mock_client_class, mock_fetch_chunks):
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
mock_client = Mock()
mock_client_class.return_value = mock_client
@@ -332,11 +344,17 @@ class TestBingAdsSource:
"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks"
)
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
def test_bing_ads_source_report_incremental(
- self, _name, last_value, mock_integrations, mock_client_class, mock_fetch_chunks
+ self, _name, last_value, mock_secrets, mock_client_class, mock_fetch_chunks
):
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
mock_client = Mock()
mock_client_class.return_value = mock_client
@@ -371,13 +389,19 @@ class TestBingAdsSource:
"products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.fetch_data_in_yearly_chunks"
)
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.BingAdsClient")
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
def test_bing_ads_source_first_sync_caps_lookback_to_retention(
- self, mock_integrations, mock_client_class, mock_fetch_chunks
+ self, mock_secrets, mock_client_class, mock_fetch_chunks
):
# On the first sync (no prior incremental value) the lookback is capped to Bing's 36-month
# retention window rather than reaching years back for data Bing no longer retains.
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
mock_client_class.return_value = Mock()
mock_fetch_chunks.return_value = iter([[{"CampaignId": "123"}]])
@@ -398,9 +422,15 @@ class TestBingAdsSource:
assert mock_fetch_chunks.call_args.kwargs["start_date"] == dt.date.today() - BING_ADS_REPORT_RETENTION
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_missing_developer_token(self, mock_integrations):
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = None
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_missing_developer_token(self, mock_secrets):
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
result = self._drive("campaigns")
@@ -416,16 +446,20 @@ class TestBingAdsSource:
("both_missing", "", ""),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_missing_oauth_app_credentials(self, _name, client_id, client_secret, mock_integrations):
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_missing_oauth_app_credentials(self, _name, client_id, client_secret, mock_secrets):
"""An empty OAuth app client id/secret raises a deterministic error instead of a doomed token request.
Without the guard the SDK posts a token request omitting client_id and Microsoft returns the opaque
AADSTS900144, which was being mis-surfaced as a customer "reconnect your integration" error.
"""
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
- mock_integrations.BING_ADS_CLIENT_ID = client_id
- mock_integrations.BING_ADS_CLIENT_SECRET = client_secret
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": client_id,
+ "BING_ADS_CLIENT_SECRET": client_secret,
+ }
result = self._drive("campaigns")
@@ -441,12 +475,16 @@ class TestBingAdsSource:
("empty", ""),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_non_numeric_account_id(self, _name, account_id, mock_integrations):
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_non_numeric_account_id(self, _name, account_id, mock_secrets):
"""A non-numeric account ID raises a deterministic, non-retryable error instead of a bare int() crash."""
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
- mock_integrations.BING_ADS_CLIENT_ID = "test_client_id"
- mock_integrations.BING_ADS_CLIENT_SECRET = "test_client_secret"
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
result = self._drive("campaigns", account_id=account_id)
@@ -455,11 +493,15 @@ class TestBingAdsSource:
assert "Bing Ads Account ID must be numeric" in str(result.raised)
assert result.requests == []
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.integrations")
- def test_bing_ads_source_incremental_missing_field(self, mock_integrations):
- mock_integrations.BING_ADS_DEVELOPER_TOKEN = "test_dev_token"
- mock_integrations.BING_ADS_CLIENT_ID = "test_client_id"
- mock_integrations.BING_ADS_CLIENT_SECRET = "test_client_secret"
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.bing_ads.credentials_service.get_secrets"
+ )
+ def test_bing_ads_source_incremental_missing_field(self, mock_secrets):
+ mock_secrets.return_value = {
+ "BING_ADS_DEVELOPER_TOKEN": "test_dev_token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ }
result = self._drive(
"campaign_performance_report",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py
index 284a1fefb024..12a67f4b6953 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_client.py
@@ -81,6 +81,8 @@ class TestBingAdsClient:
self.access_token = "test_access_token"
self.refresh_token = "test_refresh_token"
self.developer_token = "test_developer_token"
+ self.client_id = "test_client_id"
+ self.client_secret = "test_client_secret"
self.account_id = 12345
self.customer_id = 67890
@@ -105,7 +107,9 @@ class TestBingAdsClient:
instance.GetAccountsInfo.side_effect = accounts_for
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
result = client.list_accounts()
assert result == [
@@ -136,7 +140,9 @@ class TestBingAdsClient:
instance.GetUser.return_value = mock.MagicMock(User=mock.MagicMock(CustomerId=111))
instance.GetCustomersInfo.side_effect = _make_webfault("Server raised fault: 'Invalid client data.'", None)
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError, match="Failed to list Bing Ads accounts"):
client.list_accounts()
@@ -155,7 +161,9 @@ class TestBingAdsClient:
"invalid_grant The provided authorization grant has expired or been revoked."
)
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
client.get_customer_id()
@@ -174,7 +182,9 @@ class TestBingAdsClient:
"HTTPSConnectionPool(host='bingads.microsoft.com', port=443): Max retries exceeded"
)
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
client.get_customer_id()
@@ -197,7 +207,9 @@ class TestBingAdsClient:
mock_client_instance = mock_service_client.return_value
mock_client_instance.GetUser.side_effect = webfault
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
client.get_customer_id()
@@ -217,7 +229,9 @@ class TestBingAdsClient:
mock_client_instance = mock_service_client.return_value
mock_client_instance.GetUser.side_effect = webfault
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
client.get_customer_id()
@@ -256,7 +270,9 @@ class TestBingAdsClient:
mock_reporting.ReportingServiceManager.side_effect = fake_manager
mock_iter_pages.return_value = iter([])
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
# The report is a generator now, so the manager is only built once it is drained.
list(
client.get_performance_report(
@@ -294,7 +310,9 @@ class TestBingAdsClient:
mock_client_instance = mock_service_client.return_value
mock_client_instance.GetCampaignsByAccountId.return_value = mock_campaigns
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
result = list(client.get_campaigns(self.account_id, self.customer_id))
assert len(result) == 1
@@ -323,7 +341,9 @@ class TestBingAdsClient:
mock_client_instance = mock_service_client.return_value
mock_client_instance.GetCampaignsByAccountId.side_effect = webfault
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
list(client.get_campaigns(self.account_id, self.customer_id))
@@ -344,7 +364,9 @@ class TestBingAdsClient:
"HTTPSConnectionPool(host='bingads.microsoft.com', port=443): Max retries exceeded"
)
- client = BingAdsClient(self.access_token, self.refresh_token, self.developer_token)
+ client = BingAdsClient(
+ self.access_token, self.refresh_token, self.developer_token, self.client_id, self.client_secret
+ )
with pytest.raises(ValueError) as exc_info:
list(client.get_campaigns(self.account_id, self.customer_id))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py
index a4a24b6a7363..5a6449f26159 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py
@@ -250,8 +250,12 @@ class TestBingAdsSource:
],
)
@mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.credentials_service.get_secret",
- return_value="dev-token",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.credentials_service.get_secrets",
+ return_value={
+ "BING_ADS_DEVELOPER_TOKEN": "dev-token",
+ "BING_ADS_CLIENT_ID": "test_client_id",
+ "BING_ADS_CLIENT_SECRET": "test_client_secret",
+ },
)
@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.bing_ads.source.BingAdsClient")
@mock.patch.object(BingAdsSource, "get_oauth_integration")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
index eb78a0a43eb6..e3b264edf264 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/config.py
@@ -829,34 +829,3 @@ def str_to_optional_list(s: str | list[typing.Any] | None) -> list[str] | None:
raise TypeError(f"expected a string, list, or None, got {type(s).__name__}")
values = [value for value in values if value]
return values or None
-
-
-_DefaultType = typing.TypeVar("_DefaultType")
-
-
-def default_from_settings(
- key: str, converter: typing.Callable[[str], _DefaultType] = _noop_convert
-) -> typing.Callable[[], _DefaultType]:
- """Return a default factory that obtains the value from app settings."""
-
- def default_factory() -> _DefaultType:
- from django.conf import settings
-
- return converter(getattr(settings, key))
-
- return default_factory
-
-
-def default_from_env(
- key: str, converter: typing.Callable[[str], _DefaultType] = _noop_convert
-) -> typing.Callable[[], _DefaultType]:
- """Return a default factory that obtains the value from an env variable."""
-
- def default_factory() -> _DefaultType:
- import os
-
- value = os.environ[key]
-
- return converter(value)
-
- return default_factory
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py
index 67359dad3b7b..e906e0e1a22e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py
@@ -81,17 +81,5 @@ class GoogleAdsServiceAccountSourceConfig(config.Config):
customer_id: str = config.value(converter=clean_customer_id)
- private_key: str = config.value(
- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY")
- )
- private_key_id: str = config.value(
- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID")
- )
- client_email: str = config.value(
- default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL")
- )
- token_uri: str = config.value(default_factory=config.default_from_settings("GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI"))
- developer_token: str = config.value(default_factory=config.default_from_settings("GOOGLE_ADS_DEVELOPER_TOKEN"))
-
GoogleAdsSourceConfigUnion = GoogleAdsServiceAccountSourceConfig | GoogleAdsSourceConfig
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
index 21e58ce302c5..973c972b7c95 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
@@ -256,16 +256,25 @@ def google_ads_client(config: GoogleAdsSourceConfigUnion, team_id: int) -> Googl
client = _load_client_with_transient_retry(config_dict)
else:
+ resolved = credentials_service.get_secrets(
+ [
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI",
+ "GOOGLE_ADS_DEVELOPER_TOKEN",
+ ]
+ )
credentials = service_account.Credentials.from_service_account_info(
{
- "private_key": config.private_key,
- "private_key_id": config.private_key_id,
- "token_uri": config.token_uri,
- "client_email": config.client_email,
+ "private_key": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY"],
+ "private_key_id": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID"],
+ "token_uri": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI"],
+ "client_email": resolved["GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL"],
},
scopes=["https://www.googleapis.com/auth/adwords"],
)
- client = GoogleAdsClient(credentials=credentials, developer_token=config.developer_token)
+ client = GoogleAdsClient(credentials=credentials, developer_token=resolved["GOOGLE_ADS_DEVELOPER_TOKEN"])
return client
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py
index 6b7ddac4ebe6..4a5477756423 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/source.py
@@ -1,11 +1,12 @@
+from datetime import timedelta
from typing import Optional, cast
-from django.conf import settings
-
import gspread
import structlog
from google.auth import exceptions as google_auth_exceptions
+from posthog.cache_utils import cache_for
+
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
DataWarehouseSourceCategory,
@@ -15,6 +16,7 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfigType,
)
from products.warehouse_sources.backend.models.external_data_schema import SCHEMA_RESOURCE_ID_METADATA_KEY
+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
UNVERSIONED_API_VERSION,
FieldType,
@@ -37,6 +39,12 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
logger = structlog.get_logger(__name__)
+# The source config is built on many web requests, and this address is display text, not a secret.
+@cache_for(timedelta(minutes=5))
+def _service_account_email() -> str:
+ return credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")
+
+
@SourceRegistry.register
class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):
codebase_detection = detect(
@@ -187,16 +195,18 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):
except gspread.SpreadsheetNotFound:
# The Sheets API answers an unshared sheet with a 404, the same as a deleted one, so
# sharing is at least as likely as a wrong URL. Mirror the sync-time message.
+ service_account_email = credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")
return (
False,
"PostHog couldn't find a sheet at that URL. Check the URL, and share the sheet with our service "
- f"account ({settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}) as a Viewer.",
+ f"account ({service_account_email}) as a Viewer.",
)
except PermissionError:
+ service_account_email = credentials_service.get_secret("GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL")
return (
False,
"PostHog does not have access to this spreadsheet. Share it with our service account "
- f"({settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}) as a Viewer, then try again. "
+ f"({service_account_email}) as a Viewer, then try again. "
"See https://posthog.com/docs/cdp/sources/google-sheets for more.",
)
except gspread.exceptions.APIError as e:
@@ -237,6 +247,7 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):
@property
def get_source_config(self) -> SourceConfig:
+ service_account_email = _service_account_email()
return SourceConfig(
name=ExternalDataSourceType.GOOGLESHEETS,
category=DataWarehouseSourceCategory.PRODUCTIVITY,
@@ -255,7 +266,7 @@ class GoogleSheetsSource(SimpleSource[GoogleSheetsSourceConfig]):
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder="",
- caption=f'Share the sheet with our service account by entering **{settings.GOOGLE_SHEETS_SERVICE_ACCOUNT_CLIENT_EMAIL}** into the "Add people" field. We only require "Viewer" permissions to sync the sheet.',
+ caption=f'Share the sheet with our service account by entering **{service_account_email}** into the "Add people" field. We only require "Viewer" permissions to sync the sheet.',
secret=False,
)
],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
index cb4e079d7761..d032483343da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
@@ -4,14 +4,13 @@ from typing import Any
from urllib.parse import quote
from zoneinfo import ZoneInfo
-from django.conf import settings
-
import requests
from requests_oauthlib import OAuth1
from posthog.dataclasses import frozen
from posthog.models.integration.model import Integration
+from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -95,14 +94,13 @@ class TwitterAdsClient:
secret = integration.sensitive_config.get("oauth_token_secret")
if not token or not secret:
raise ValueError(MISSING_INTEGRATION)
- if not settings.TWITTER_ADS_CONSUMER_KEY or not settings.TWITTER_ADS_CONSUMER_SECRET:
+ consumer = credentials_service.get_secrets(["TWITTER_ADS_CONSUMER_KEY", "TWITTER_ADS_CONSUMER_SECRET"])
+ consumer_key = consumer["TWITTER_ADS_CONSUMER_KEY"]
+ consumer_secret = consumer["TWITTER_ADS_CONSUMER_SECRET"]
+ if not consumer_key or not consumer_secret:
raise ValueError(MISSING_APP)
- self.session = make_tracked_session(
- allow_redirects=False, redact_values=(token, secret, settings.TWITTER_ADS_CONSUMER_SECRET)
- )
- self.session.auth = OAuth1(
- settings.TWITTER_ADS_CONSUMER_KEY, settings.TWITTER_ADS_CONSUMER_SECRET, token, secret
- )
+ self.session = make_tracked_session(allow_redirects=False, redact_values=(token, secret, consumer_secret))
+ self.session.auth = OAuth1(consumer_key, consumer_secret, token, secret)
self.base_url = f"https://ads-api.x.com/{api_version}"
def get(self, path: str, params: dict[str, str | int] | None = None) -> dict[str, Any]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
index 198a4a78ac31..285678eb86d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
@@ -6,8 +6,6 @@ from dataclasses import dataclass
import pytest
from unittest import mock
-from django.test import override_settings
-
import pyarrow as pa
from posthog.models import Team
@@ -62,78 +60,40 @@ def developer_token() -> str:
@pytest.fixture
-def service_account_config() -> dict[str, str]:
- """Return a Service Account configuration dictionary to use in tests."""
+def service_account_secrets(developer_token: str):
+ """Serve the platform service account credentials through the credentials service wrapper."""
credentials_file_path = os.environ["GOOGLE_SERVICE_ACCOUNT_CREDENTIALS"]
with open(credentials_file_path) as f:
credentials = json.load(f)
- return {
- "private_key": credentials["private_key"],
- "private_key_id": credentials["private_key_id"],
- "token_uri": credentials["token_uri"],
- "client_email": credentials["client_email"],
- }
-
-
-def test_google_ads_source_config_loads(customer_id: str, developer_token: str):
- private_key = "private_key"
- private_key_id = "id"
- client_email = "[email protected]"
- token_uri = "https://posthog.com"
-
- job_inputs = {
- "resource_name": "campaign",
- "customer_id": customer_id,
- }
-
- with override_settings(
- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY=private_key,
- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID=private_key_id,
- GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL=client_email,
- GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI=token_uri,
- GOOGLE_ADS_DEVELOPER_TOKEN=developer_token,
- ):
- cfg = GoogleAdsServiceAccountSourceConfig.from_dict(job_inputs)
-
- assert cfg.private_key == private_key
- assert cfg.private_key_id == private_key_id
- assert cfg.client_email == client_email
- assert cfg.token_uri == token_uri
- assert cfg.developer_token == developer_token
- assert cfg.customer_id == customer_id
-
-
-def test_google_ads_source_config_handles_customer_id_with_dashes(developer_token: str):
- private_key = "private_key"
- private_key_id = "id"
- client_email = "[email protected]"
- token_uri = "https://posthog.com"
-
- job_inputs = {
- "resource_name": "campaign",
- "customer_id": "111-111-1111",
- }
-
- with override_settings(
- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY=private_key,
- GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID=private_key_id,
- GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL=client_email,
- GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI=token_uri,
- GOOGLE_ADS_DEVELOPER_TOKEN=developer_token,
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.credentials_service.get_secrets",
+ return_value={
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY": credentials["private_key"],
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID": credentials["private_key_id"],
+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI": credentials["token_uri"],
+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL": credentials["client_email"],
+ "GOOGLE_ADS_DEVELOPER_TOKEN": developer_token,
+ },
):
- cfg = GoogleAdsServiceAccountSourceConfig.from_dict(job_inputs)
+ yield
+
+
+ "raw_customer_id,expected",
+ [
+ ("1111111111", "1111111111"),
+ ("111-111-1111", "1111111111"),
+ ],
+)
+def test_google_ads_service_account_config_loads(raw_customer_id: str, expected: str):
+ cfg = GoogleAdsServiceAccountSourceConfig.from_dict({"resource_name": "campaign", "customer_id": raw_customer_id})
- assert cfg.private_key == private_key
- assert cfg.private_key_id == private_key_id
- assert cfg.client_email == client_email
- assert cfg.token_uri == token_uri
- assert cfg.developer_token == developer_token
- assert cfg.customer_id == "1111111111"
+ assert cfg.customer_id == expected
@SKIP_IF_MISSING_GOOGLE_ADS_CREDENTIALS
-def test_get_schemas(customer_id: str, developer_token: str, service_account_config: dict[str, str]):
+def test_get_schemas(customer_id: str, service_account_secrets: None):
"""Test get_schemas returns well-known schemas.
This test is not exhaustive and merely limits itself to asserting a handful
@@ -142,9 +102,7 @@ def test_get_schemas(customer_id: str, developer_token: str, service_account_con
org = Organization.objects.create(name="org")
team = Team.objects.create(organization=org)
- cfg = GoogleAdsServiceAccountSourceConfig(
- customer_id=customer_id, developer_token=developer_token, **service_account_config
- )
+ cfg = GoogleAdsServiceAccountSourceConfig(customer_id=customer_id)
schemas = get_schemas(cfg, team_id=team.id, api_version=GoogleAdsSource().default_version)
assert "campaign" in schemas
@@ -171,7 +129,7 @@ def test_get_schemas(customer_id: str, developer_token: str, service_account_con
@SKIP_IF_MISSING_GOOGLE_ADS_CREDENTIALS
-def test_google_ads_source(customer_id: str, developer_token: str, service_account_config: dict[str, str]):
+def test_google_ads_source(customer_id: str, service_account_secrets: None):
"""Test google_ads_source can iterate rows.
This test is not exhaustive and merely limits itself to attempting to
@@ -181,9 +139,7 @@ def test_google_ads_source(customer_id: str, developer_token: str, service_accou
org = Organization.objects.create(name="org")
team = Team.objects.create(organization=org)
- cfg = GoogleAdsServiceAccountSourceConfig(
- customer_id=customer_id, developer_token=developer_token, **service_account_config
- )
+ cfg = GoogleAdsServiceAccountSourceConfig(customer_id=customer_id)
for resource in (
"campaign",
"campaign_stats",
@@ -605,6 +561,9 @@ class TestGoogleAdsClientStaleConnection:
call_order = [name for name, _, _ in manager.mock_calls]
assert call_order.index("close_old_connections") < call_order.index("integration_get")
+ @mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.credentials_service.get_secrets"
+ )
@mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.service_account"
)
@@ -615,20 +574,28 @@ class TestGoogleAdsClientStaleConnection:
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.close_old_connections"
)
def test_service_account_client_does_not_touch_django_connections(
- self, mock_close_old_connections, mock_client_cls, mock_service_account
+ self, mock_close_old_connections, mock_client_cls, mock_service_account, mock_get_secrets
):
- config = GoogleAdsServiceAccountSourceConfig(
- customer_id="1234567890",
- private_key="pk",
- private_key_id="pk_id",
- client_email="[email protected]",
- token_uri="https://example.com",
- developer_token="dev",
- )
+ mock_get_secrets.return_value = {
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY": "pk",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_PRIVATE_KEY_ID": "pk_id",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_CLIENT_EMAIL": "[email protected]",
+ "GOOGLE_ADS_SERVICE_ACCOUNT_TOKEN_URI": "https://example.com",
+ "GOOGLE_ADS_DEVELOPER_TOKEN": "dev",
+ }
+ config = GoogleAdsServiceAccountSourceConfig(customer_id="1234567890")
google_ads_client(config, team_id=7)
mock_close_old_connections.assert_not_called()
+ mock_service_account.Credentials.from_service_account_info.assert_called_once()
+ assert mock_service_account.Credentials.from_service_account_info.call_args.args[0] == {
+ "private_key": "pk",
+ "private_key_id": "pk_id",
+ "token_uri": "https://example.com",
+ "client_email": "[email protected]",
+ }
+ assert mock_client_cls.call_args.kwargs["developer_token"] == "dev"
def test_source_registration_does_not_import_google_ads_sdk(google_ads_sdk_leak_check) -> None: