:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(bigquery): stop blaming customer IAM when posthog gcp credentials fail (#121300)
Tom Owerse216d31b2869
diff --git a/products/batch_exports/backend/facade/destinations/bigquery.py b/products/batch_exports/backend/facade/destinations/bigquery.pyindex 26dc3f5cecf8..a7bb7d9d27e2 100644--- a/products/batch_exports/backend/facade/destinations/bigquery.py+++ b/products/batch_exports/backend/facade/destinations/bigquery.py@@ -14,18 +14,22 @@ Importing this module loads the destination's vendor SDK, so keep it off thefrom products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (BigQueryClient,+ GoogleCloudCredentialsError,MissingRequiredPermissionsError,ServiceAccountNotFoundError,ServiceAccountOwnershipError,+ ensure_our_google_cloud_credentials_are_valid,get_our_google_cloud_credentials,verify_impersonated_service_account_ownership,)__all__ = ["BigQueryClient",+ "GoogleCloudCredentialsError","MissingRequiredPermissionsError","ServiceAccountNotFoundError","ServiceAccountOwnershipError",+ "ensure_our_google_cloud_credentials_are_valid","get_our_google_cloud_credentials","verify_impersonated_service_account_ownership",]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.pyindex 4635dbcb4aaf..efe74cf69507 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py@@ -265,6 +265,16 @@ BIGQUERY_IMPERSONATION_PERMISSION_ERROR = ("the source.")+# Raised when the PostHog worker cannot get PostHog's own Google Cloud credentials, the first hop of+# the impersonation chain. The customer's service account is never contacted, so no change on their+# side can fix it. Deliberately absent from `get_non_retryable_errors`, so the sync retries and the+# failure still reaches error tracking.+BIGQUERY_POSTHOG_CREDENTIALS_ERROR = (+ "PostHog could not get its own Google Cloud credentials, so it could not connect to your "+ "service account. This is a problem on PostHog's side, and you don't need to change your service "+ "account. Syncs retry automatically. If you are setting up the source, try again in a few minutes."+)+# Stable prefix of the ownership failure raised when the service account description does not name# the connecting organization. The rest of the message carries the organization ID the user must add.BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX = "Could not verify that service account"@@ -662,9 +672,11 @@ def _impersonated_credentials(service_account_email: str, team_id: int) -> googl# Imported here so the source registry — which the API imports on every request path — does not# pull in the batch-export Temporal module.from products.batch_exports.backend.facade.destinations.bigquery import ( # noqa: PLC0415 — keeps the batch-export Temporal module off the source registry's import path+ GoogleCloudCredentialsError,MissingRequiredPermissionsError,ServiceAccountNotFoundError,ServiceAccountOwnershipError,+ ensure_our_google_cloud_credentials_are_valid,get_our_google_cloud_credentials,verify_impersonated_service_account_ownership,)@@ -672,6 +684,15 @@ def _impersonated_credentials(service_account_email: str, team_id: int) -> googlif not settings.BATCH_EXPORT_BIGQUERY_STS_AUDIENCE_FIELD or not settings.BATCH_EXPORT_BIGQUERY_SERVICE_ACCOUNT:raise BigQueryAuthResolutionError(BIGQUERY_IMPERSONATION_UNAVAILABLE_ERROR)+ # The ownership check below authenticates with PostHog's own credentials. If this worker cannot+ # get them, the check fails with the same "getAccessToken denied" text that a missing grant on+ # the customer's service account produces, and the sync tells the customer to fix IAM they+ # already set up. Batch exports run the same check first for the same reason.+ try:+ async_to_sync(ensure_our_google_cloud_credentials_are_valid)()+ except GoogleCloudCredentialsError as e:+ raise BigQueryAuthResolutionError(BIGQUERY_POSTHOG_CREDENTIALS_ERROR) from e+# Confirm the team owns the account before impersonating it, or a team that merely knows another# org's service account email could have PostHog read that org's data on its behalf. Batch# exports and the BigQuery destination writer guard the same way at the same point.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.pyindex d830b43a68b1..9e615fe33628 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py@@ -29,7 +29,10 @@ from requests.adapters import HTTPAdapterfrom posthog.models.integration import Integrationfrom posthog.models.team.team import Team-from products.batch_exports.backend.facade.destinations.bigquery import ServiceAccountOwnershipError+from products.batch_exports.backend.facade.destinations.bigquery import (+ GoogleCloudCredentialsError,+ ServiceAccountOwnershipError,+)from products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errorsfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigquery import bigquery as bq_modulefrom products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.bigquery import (@@ -45,6 +48,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.bBIGQUERY_INVALID_TOKEN_URI_ERROR,BIGQUERY_MISSING_KEY_FILE_FIELDS_ERROR,BIGQUERY_NO_CREDENTIALS_ERROR,+ BIGQUERY_POSTHOG_CREDENTIALS_ERROR,BIGQUERY_QUERY_CREATE_RETRY,BIGQUERY_QUERY_JOB_RETRY,BIGQUERY_READ_ROWS_RETRY,@@ -1653,6 +1657,7 @@ def test_bigquery_auth_from_keyless_integration_impersonates_only_after_ownershiconfig = BigQuerySourceConfig(dataset_id="my_dataset", auth_type=_integration_auth(integration.id))with (+ mock.patch(f"{_BATCH_EXPORT_MODULE}.ensure_our_google_cloud_credentials_are_valid"),mock.patch(f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership") as mock_verify,mock.patch(f"{_BATCH_EXPORT_MODULE}.get_our_google_cloud_credentials"),mock.patch.object(bq_module.google_auth_impersonated_credentials, "Credentials") as mock_impersonated,@@ -1665,30 +1670,61 @@ def test_bigquery_auth_from_keyless_integration_impersonates_only_after_ownershiassert mock_impersonated.call_args.kwargs["target_principal"] == "[email protected]"+# Verbatim wording from batch exports, which raises the same error for the same check.+_OWNERSHIP_ERROR = ServiceAccountOwnershipError("[email protected]", "org-uuid")+++ "own_credentials_error, ownership_error, expected_message, expect_non_retryable, expect_ownership_checked",+ [+ # The customer has to change their service account description, so the sync must stop+ # rather than retry the same rejection every run.+ pytest.param(None, _OWNERSHIP_ERROR, str(_OWNERSHIP_ERROR), True, True, id="ownership_unverified"),+ # PostHog's worker cannot get its own Google Cloud credentials. The ownership check would+ # fail with the same "getAccessToken denied" text as a missing customer grant, which is+ # non-retryable and tells the customer to fix IAM. Nothing on their side can fix this.+ pytest.param(+ GoogleCloudCredentialsError(),+ None,+ BIGQUERY_POSTHOG_CREDENTIALS_ERROR,+ False,+ False,+ id="posthog_credentials_unavailable",+ ),+ ],+)@pytest.mark.django_db-def test_bigquery_unverified_service_account_ownership_is_reported_and_not_retried(team, settings):+def test_bigquery_impersonation_precheck_failure_is_reported_and_retried_only_on_posthog_side(+ team,+ settings,+ own_credentials_error,+ ownership_error,+ expected_message,+ expect_non_retryable,+ expect_ownership_checked,+):settings.BATCH_EXPORT_BIGQUERY_STS_AUDIENCE_FIELD = "//iam.googleapis.com/projects/1/locations/global"settings.BATCH_EXPORT_BIGQUERY_SERVICE_ACCOUNT = "[email protected]"integration = _google_cloud_integration(team, with_key=False)config = BigQuerySourceConfig(dataset_id="my_dataset", auth_type=_integration_auth(integration.id))- # Verbatim wording from batch exports, which raises the same error for the same check.- ownership_error = ServiceAccountOwnershipError("[email protected]", "org-uuid")with (mock.patch(- f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership", side_effect=ownership_error+ f"{_BATCH_EXPORT_MODULE}.ensure_our_google_cloud_credentials_are_valid", side_effect=own_credentials_error),+ mock.patch(+ f"{_BATCH_EXPORT_MODULE}.verify_impersonated_service_account_ownership", side_effect=ownership_error+ ) as mock_verify,mock.patch(f"{_BATCH_EXPORT_MODULE}.get_our_google_cloud_credentials"),mock.patch.object(bq_module, "bigquery_client") as mock_client,):ok, message = BigQuerySource().validate_credentials(config, team_id=team.id)assert ok is False- assert message == str(ownership_error)+ assert message == expected_messagemock_client.assert_not_called()- # The customer has to change their service account description, so the sync must stop rather- # than retry the same rejection every run.- assert any(key in str(ownership_error) for key in BigQuerySource().get_non_retryable_errors())+ assert mock_verify.called is expect_ownership_checked+ assert error_message_matches(expected_message, BigQuerySource().get_non_retryable_errors()) is expect_non_retryable@pytest.mark.django_db