Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(bigquery): stop blaming customer IAM when posthog gcp credentials fail (#121300)

Tom Owerse216d31b2869
diff --git a/products/batch_exports/backend/facade/destinations/bigquery.py b/products/batch_exports/backend/facade/destinations/bigquery.py
index 26dc3f5cecf8..a7bb7d9d27e2 100644
--- a/products/batch_exports/backend/facade/destinations/bigquery.py
+++ b/products/batch_exports/backend/facade/destinations/bigquery.py
@@ -14,18 +14,22 @@ Importing this module loads the destination's vendor SDK, so keep it off the
from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (
BigQueryClient,
+ GoogleCloudCredentialsError,
MissingRequiredPermissionsError,
ServiceAccountNotFoundError,
ServiceAccountOwnershipError,
+ ensure_our_google_cloud_credentials_are_valid,
get_our_google_cloud_credentials,
verify_impersonated_service_account_ownership,
)
__all__ = [
"BigQueryClient",
+ "GoogleCloudCredentialsError",
"MissingRequiredPermissionsError",
"ServiceAccountNotFoundError",
"ServiceAccountOwnershipError",
+ "ensure_our_google_cloud_credentials_are_valid",
"get_our_google_cloud_credentials",
"verify_impersonated_service_account_ownership",
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
index 4635dbcb4aaf..efe74cf69507 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
@@ -265,6 +265,16 @@ BIGQUERY_IMPERSONATION_PERMISSION_ERROR = (
"the source."
)
+# Raised when the PostHog worker cannot get PostHog's own Google Cloud credentials, the first hop of
+# the impersonation chain. The customer's service account is never contacted, so no change on their
+# side can fix it. Deliberately absent from `get_non_retryable_errors`, so the sync retries and the
+# failure still reaches error tracking.
+BIGQUERY_POSTHOG_CREDENTIALS_ERROR = (
+ "PostHog could not get its own Google Cloud credentials, so it could not connect to your "
+ "service account. This is a problem on PostHog's side, and you don't need to change your service "
+ "account. Syncs retry automatically. If you are setting up the source, try again in a few minutes."
+)
+
# Stable prefix of the ownership failure raised when the service account description does not name
# the connecting organization. The rest of the message carries the organization ID the user must add.
BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX = "Could not verify that service account"
@@ -662,9 +672,11 @@ def _impersonated_credentials(service_account_email: str, team_id: int) -> googl
# Imported here so the source registry — which the API imports on every request path — does not
# pull in the batch-export Temporal module.
from products.batch_exports.backend.facade.destinations.bigquery import ( # noqa: PLC0415 — keeps the batch-export Temporal module off the source registry's import path
+ GoogleCloudCredentialsError,
MissingRequiredPermissionsError,
ServiceAccountNotFoundError,
ServiceAccountOwnershipError,
+ ensure_our_google_cloud_credentials_are_valid,
get_our_google_cloud_credentials,
verify_impersonated_service_account_ownership,
)
@@ -672,6 +684,15 @@ def _impersonated_credentials(service_account_email: str, team_id: int) -> googl
if not settings.BATCH_EXPORT_BIGQUERY_STS_AUDIENCE_FIELD or not settings.BATCH_EXPORT_BIGQUERY_SERVICE_ACCOUNT:
raise BigQueryAuthResolutionError(BIGQUERY_IMPERSONATION_UNAVAILABLE_ERROR)
+ # The ownership check below authenticates with PostHog's own credentials. If this worker cannot
+ # get them, the check fails with the same "getAccessToken denied" text that a missing grant on
+ # the customer's service account produces, and the sync tells the customer to fix IAM they
+ # already set up. Batch exports run the same check first for the same reason.
+ try:
+ async_to_sync(ensure_our_google_cloud_credentials_are_valid)()
+ except GoogleCloudCredentialsError as e:
+ raise BigQueryAuthResolutionError(BIGQUERY_POSTHOG_CREDENTIALS_ERROR) from e
+
# Confirm the team owns the account before impersonating it, or a team that merely knows another
# org's service account email could have PostHog read that org's data on its behalf. Batch
# exports and the BigQuery destination writer guard the same way at the same point.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
index d830b43a68b1..9e615fe33628 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
@@ -29,7 +29,10 @@ from requests.adapters import HTTPAdapter
from posthog.models.integration import Integration
from posthog.models.team.team import Team
-from products.batch_exports.backend.facade.destinations.bigquery import ServiceAccountOwnershipError
+from products.batch_exports.backend.facade.destinations.bigquery import (
+ GoogleCloudCredentialsError,
+ ServiceAccountOwnershipError,
+)
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errors
from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery import bigquery as bq_module
from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.bigquery import (
@@ -45,6 +48,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.b
BIGQUERY_INVALID_TOKEN_URI_ERROR,
BIGQUERY_MISSING_KEY_FILE_FIELDS_ERROR,
BIGQUERY_NO_CREDENTIALS_ERROR,
+ BIGQUERY_POSTHOG_CREDENTIALS_ERROR,
BIGQUERY_QUERY_CREATE_RETRY,
BIGQUERY_QUERY_JOB_RETRY,
BIGQUERY_READ_ROWS_RETRY,
@@ -1653,6 +1657,7 @@ def test_bigquery_auth_from_keyless_integration_impersonates_only_after_ownershi
config = BigQuerySourceConfig(dataset_id="my_dataset", auth_type=_integration_auth(integration.id))
with (
+ mock.patch(f"{_BATCH_EXPORT_MODULE}.ensure_our_google_cloud_credentials_are_valid"),
mock.patch(f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership") as mock_verify,
mock.patch(f"{_BATCH_EXPORT_MODULE}.get_our_google_cloud_credentials"),
mock.patch.object(bq_module.google_auth_impersonated_credentials, "Credentials") as mock_impersonated,
@@ -1665,30 +1670,61 @@ def test_bigquery_auth_from_keyless_integration_impersonates_only_after_ownershi
assert mock_impersonated.call_args.kwargs["target_principal"] == "[email protected]"
+# Verbatim wording from batch exports, which raises the same error for the same check.
+_OWNERSHIP_ERROR = ServiceAccountOwnershipError("[email protected]", "org-uuid")
+
+
+ "own_credentials_error, ownership_error, expected_message, expect_non_retryable, expect_ownership_checked",
+ [
+ # The customer has to change their service account description, so the sync must stop
+ # rather than retry the same rejection every run.
+ pytest.param(None, _OWNERSHIP_ERROR, str(_OWNERSHIP_ERROR), True, True, id="ownership_unverified"),
+ # PostHog's worker cannot get its own Google Cloud credentials. The ownership check would
+ # fail with the same "getAccessToken denied" text as a missing customer grant, which is
+ # non-retryable and tells the customer to fix IAM. Nothing on their side can fix this.
+ pytest.param(
+ GoogleCloudCredentialsError(),
+ None,
+ BIGQUERY_POSTHOG_CREDENTIALS_ERROR,
+ False,
+ False,
+ id="posthog_credentials_unavailable",
+ ),
+ ],
+)
@pytest.mark.django_db
-def test_bigquery_unverified_service_account_ownership_is_reported_and_not_retried(team, settings):
+def test_bigquery_impersonation_precheck_failure_is_reported_and_retried_only_on_posthog_side(
+ team,
+ settings,
+ own_credentials_error,
+ ownership_error,
+ expected_message,
+ expect_non_retryable,
+ expect_ownership_checked,
+):
settings.BATCH_EXPORT_BIGQUERY_STS_AUDIENCE_FIELD = "//iam.googleapis.com/projects/1/locations/global"
settings.BATCH_EXPORT_BIGQUERY_SERVICE_ACCOUNT = "[email protected]"
integration = _google_cloud_integration(team, with_key=False)
config = BigQuerySourceConfig(dataset_id="my_dataset", auth_type=_integration_auth(integration.id))
- # Verbatim wording from batch exports, which raises the same error for the same check.
- ownership_error = ServiceAccountOwnershipError("[email protected]", "org-uuid")
with (
mock.patch(
- f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership", side_effect=ownership_error
+ f"{_BATCH_EXPORT_MODULE}.ensure_our_google_cloud_credentials_are_valid", side_effect=own_credentials_error
),
+ mock.patch(
+ f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership", side_effect=ownership_error
+ ) as mock_verify,
mock.patch(f"{_BATCH_EXPORT_MODULE}.get_our_google_cloud_credentials"),
mock.patch.object(bq_module, "bigquery_client") as mock_client,
):
ok, message = BigQuerySource().validate_credentials(config, team_id=team.id)
assert ok is False
- assert message == str(ownership_error)
+ assert message == expected_message
mock_client.assert_not_called()
- # The customer has to change their service account description, so the sync must stop rather
- # than retry the same rejection every run.
- assert any(key in str(ownership_error) for key in BigQuerySource().get_non_retryable_errors())
+ assert mock_verify.called is expect_ownership_checked
+ assert error_message_matches(expected_message, BigQuerySource().get_non_retryable_errors()) is expect_non_retryable
@pytest.mark.django_db