:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(bigquery): classify VPC Service Controls denials as non-retryable (#120516)
Tom Owers99905d22aad8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.pyindex 87960dabb175..4635dbcb4aaf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py@@ -269,6 +269,19 @@ BIGQUERY_IMPERSONATION_PERMISSION_ERROR = (# the connecting organization. The rest of the message carries the organization ID the user must add.BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX = "Could not verify that service account"+# Raised as a `Forbidden` (403) by every BigQuery REST/gRPC call when a VPC Service Controls+# perimeter in the customer's Google Cloud organization blocks the request outright, e.g. "VPC+# Service Controls: Request is prohibited by organization's policy. vpcServiceControlsUniqueIdentifier:+# ...". This is an org-level network policy the customer's Google Cloud admin controls, not an IAM+# role we can tell them to grant — retrying the same request only repeats the same denial. Shared+# between onboarding credential validation and the sync-path classifier in+# `BigQuerySource.get_non_retryable_errors`, so the two stay in lockstep.+BIGQUERY_VPC_SERVICE_CONTROLS_ERROR = (+ "A VPC Service Controls policy in your Google Cloud organization is blocking this request to "+ "BigQuery. Add an access level for PostHog to the perimeter around this project (or the "+ "dataset's project), or exempt BigQuery from the perimeter, then reconnect the source."+)+class BigQueryAuthResolutionError(Exception):"""Carries a user-safe explanation of why a source's credentials could not be resolved."""@@ -988,6 +1001,8 @@ def classify_bigquery_validation_error(e: Exception) -> str:return BIGQUERY_INVALID_IDENTIFIER_ERRORif "was not found in location" in message or "Not found: Dataset" in message:return BIGQUERY_DATASET_NOT_FOUND_ERROR+ if "VPC Service Controls" in message:+ return BIGQUERY_VPC_SERVICE_CONTROLS_ERRORif "Access Denied" in message or "PermissionDenied" in message or "permission denied" in message:return BIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR# Genuinely unexpected — keep the signal, and fall back to a generic message so no rawdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.pyindex 05504f30df18..c0cc331829dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.bBIGQUERY_RESOURCES_EXCEEDED_ERROR,BIGQUERY_SERVICE_ACCOUNT_NOT_FOUND_ERROR,BIGQUERY_TOKEN_RESPONSE_ERROR,+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,BigQueryImplementation,build_destination_table_prefix,classify_bigquery_validation_error,@@ -315,6 +316,8 @@ class BigQuerySource(SQLSource[BigQuerySourceConfig]):"iam.serviceAccounts.getAccessToken' denied": BIGQUERY_IMPERSONATION_PERMISSION_ERROR,"iam.serviceAccounts.getAccessToken\\' denied": BIGQUERY_IMPERSONATION_PERMISSION_ERROR,BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX: None,+ # Matched on the stable wording, not the volatile vpcServiceControlsUniqueIdentifier.+ "VPC Service Controls": BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,# Writing query results into the `__posthog_import_...` temp tables PostHog creates# (`WRITE_TRUNCATE` in `_run_destination_query_with_job_retry`, on incremental / view /# row-filtered reads) needs write access on the dataset those tables live in. When thediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.pyindex df149aa7f467..7c2ae0671550 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.bBIGQUERY_TOKEN_RESPONSE_ERROR,BIGQUERY_VALIDATION_GENERIC_ERROR,BIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR,+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,BigQueryAuthResolutionError,BigQueryCredentialsRejectedError,BigQueryDatasetNotFoundError,@@ -698,6 +699,24 @@ def test_bigquery_unparseable_private_key_is_non_retryable(observed_error):assert all(non_retryable_errors[key] is not None for key in matching)+ "observed_error",+ [+ # Volatile project/dataset/identifier must not be required for the match.+ "GET https://bigquery.googleapis.com/bigquery/v2/projects/my-project/datasets/my_dataset/"+ "tables?prettyPrint=false: VPC Service Controls: Request is prohibited by organization's "+ "policy. vpcServiceControlsUniqueIdentifier: abc123.",+ "403 VPC Service Controls: Request is prohibited by organization's policy. "+ "vpcServiceControlsUniqueIdentifier: xyz789.",+ ],+)+def test_non_retryable_errors_match_vpc_service_controls_denial(observed_error):+ non_retryable_errors = BigQuerySource().get_non_retryable_errors()+ matching = [key for key in non_retryable_errors if key in observed_error]+ assert matching, "VPC Service Controls denial should be recognised as non-retryable"+ assert all(non_retryable_errors[key] is not None for key in matching)++def _run_delete_all_temp_destination_tables(side_effect, logger):bq = mock.MagicMock()bq.list_tables.side_effect = side_effect@@ -949,6 +968,15 @@ def test_bigquery_rejects_non_google_token_uri_before_building_credentials(tokenBIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR,False,),+ (+ Forbidden(+ "GET https://bigquery.googleapis.com/bigquery/v2/projects/my-project/datasets/"+ "my_dataset/tables?prettyPrint=false: VPC Service Controls: Request is prohibited "+ "by organization's policy. vpcServiceControlsUniqueIdentifier: abc123."+ ),+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,+ False,+ ),(RuntimeError("something unexpected"), BIGQUERY_VALIDATION_GENERIC_ERROR, True),],)