Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(bigquery): classify VPC Service Controls denials as non-retryable (#120516)

Tom Owers99905d22aad8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
index 87960dabb175..4635dbcb4aaf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
@@ -269,6 +269,19 @@ BIGQUERY_IMPERSONATION_PERMISSION_ERROR = (
# the connecting organization. The rest of the message carries the organization ID the user must add.
BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX = "Could not verify that service account"
+# Raised as a `Forbidden` (403) by every BigQuery REST/gRPC call when a VPC Service Controls
+# perimeter in the customer's Google Cloud organization blocks the request outright, e.g. "VPC
+# Service Controls: Request is prohibited by organization's policy. vpcServiceControlsUniqueIdentifier:
+# ...". This is an org-level network policy the customer's Google Cloud admin controls, not an IAM
+# role we can tell them to grant — retrying the same request only repeats the same denial. Shared
+# between onboarding credential validation and the sync-path classifier in
+# `BigQuerySource.get_non_retryable_errors`, so the two stay in lockstep.
+BIGQUERY_VPC_SERVICE_CONTROLS_ERROR = (
+ "A VPC Service Controls policy in your Google Cloud organization is blocking this request to "
+ "BigQuery. Add an access level for PostHog to the perimeter around this project (or the "
+ "dataset's project), or exempt BigQuery from the perimeter, then reconnect the source."
+)
+
class BigQueryAuthResolutionError(Exception):
"""Carries a user-safe explanation of why a source's credentials could not be resolved."""
@@ -988,6 +1001,8 @@ def classify_bigquery_validation_error(e: Exception) -> str:
return BIGQUERY_INVALID_IDENTIFIER_ERROR
if "was not found in location" in message or "Not found: Dataset" in message:
return BIGQUERY_DATASET_NOT_FOUND_ERROR
+ if "VPC Service Controls" in message:
+ return BIGQUERY_VPC_SERVICE_CONTROLS_ERROR
if "Access Denied" in message or "PermissionDenied" in message or "permission denied" in message:
return BIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR
# Genuinely unexpected — keep the signal, and fall back to a generic message so no raw
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py
index 05504f30df18..c0cc331829dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/source.py
@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.b
BIGQUERY_RESOURCES_EXCEEDED_ERROR,
BIGQUERY_SERVICE_ACCOUNT_NOT_FOUND_ERROR,
BIGQUERY_TOKEN_RESPONSE_ERROR,
+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,
BigQueryImplementation,
build_destination_table_prefix,
classify_bigquery_validation_error,
@@ -315,6 +316,8 @@ class BigQuerySource(SQLSource[BigQuerySourceConfig]):
"iam.serviceAccounts.getAccessToken' denied": BIGQUERY_IMPERSONATION_PERMISSION_ERROR,
"iam.serviceAccounts.getAccessToken\\' denied": BIGQUERY_IMPERSONATION_PERMISSION_ERROR,
BIGQUERY_OWNERSHIP_UNVERIFIED_ERROR_PREFIX: None,
+ # Matched on the stable wording, not the volatile vpcServiceControlsUniqueIdentifier.
+ "VPC Service Controls": BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,
# Writing query results into the `__posthog_import_...` temp tables PostHog creates
# (`WRITE_TRUNCATE` in `_run_destination_query_with_job_retry`, on incremental / view /
# row-filtered reads) needs write access on the dataset those tables live in. When the
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
index df149aa7f467..7c2ae0671550 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/tests/test_source.py
@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.b
BIGQUERY_TOKEN_RESPONSE_ERROR,
BIGQUERY_VALIDATION_GENERIC_ERROR,
BIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR,
+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,
BigQueryAuthResolutionError,
BigQueryCredentialsRejectedError,
BigQueryDatasetNotFoundError,
@@ -698,6 +699,24 @@ def test_bigquery_unparseable_private_key_is_non_retryable(observed_error):
assert all(non_retryable_errors[key] is not None for key in matching)
+ "observed_error",
+ [
+ # Volatile project/dataset/identifier must not be required for the match.
+ "GET https://bigquery.googleapis.com/bigquery/v2/projects/my-project/datasets/my_dataset/"
+ "tables?prettyPrint=false: VPC Service Controls: Request is prohibited by organization's "
+ "policy. vpcServiceControlsUniqueIdentifier: abc123.",
+ "403 VPC Service Controls: Request is prohibited by organization's policy. "
+ "vpcServiceControlsUniqueIdentifier: xyz789.",
+ ],
+)
+def test_non_retryable_errors_match_vpc_service_controls_denial(observed_error):
+ non_retryable_errors = BigQuerySource().get_non_retryable_errors()
+ matching = [key for key in non_retryable_errors if key in observed_error]
+ assert matching, "VPC Service Controls denial should be recognised as non-retryable"
+ assert all(non_retryable_errors[key] is not None for key in matching)
+
+
def _run_delete_all_temp_destination_tables(side_effect, logger):
bq = mock.MagicMock()
bq.list_tables.side_effect = side_effect
@@ -949,6 +968,15 @@ def test_bigquery_rejects_non_google_token_uri_before_building_credentials(token
BIGQUERY_VALIDATION_PERMISSION_DENIED_ERROR,
False,
),
+ (
+ Forbidden(
+ "GET https://bigquery.googleapis.com/bigquery/v2/projects/my-project/datasets/"
+ "my_dataset/tables?prettyPrint=false: VPC Service Controls: Request is prohibited "
+ "by organization's policy. vpcServiceControlsUniqueIdentifier: abc123."
+ ),
+ BIGQUERY_VPC_SERVICE_CONTROLS_ERROR,
+ False,
+ ),
(RuntimeError("something unexpected"), BIGQUERY_VALIDATION_GENERIC_ERROR, True),
],
)