Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Introduce billing-project for BigQuery (#79489)

metambene6216478c671
diff --git a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
index 58077bf2db1..823b241a221 100644
--- a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
+++ b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
@@ -32,7 +32,7 @@ const databaseTestCases = [
connectionString:
"jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=MyBigQueryProject;OAuthType=1;",
expectedFields: [
- { label: "Project ID (override)", value: "MyBigQueryProject" },
+ { label: "Data project ID", value: "MyBigQueryProject" },
{ label: "Display name", value: "MyBigQueryProject" },
],
},
diff --git a/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml
index 039748659d3..3de0385175b 100644
--- a/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml
+++ b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml
@@ -7,11 +7,17 @@ driver:
display-name: BigQuery
lazy-load: true
connection-properties:
+ - name: billing-project-id
+ display-name: Billing project ID
+ helper-text: Google Cloud project where query jobs run and are billed. Defaults to the service account's project.
+ placeholder: my-billing-project
+ required: false
+ type: string
- name: project-id
- display-name: Project ID (override)
- helper-text: Project ID to be used for authentication. You can omit this field if you are only querying datasets owned by your organization.
+ display-name: Data project ID
+ helper-text: Google Cloud project that owns the datasets you want to query. Defaults to the service account's project.
required: false
- placeholder: 1w08oDRKPrOqBt06yxY8uiCz2sSvOp3u
+ placeholder: my-warehouse-project
- name: service-account-json
display-name: Service account JSON file
helper-text: This JSON file contains the credentials Metabase needs to read and query your dataset.
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 78a22764102..dc7631e82f3 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -130,6 +130,9 @@
(.setUniverseDomain bq-bldr universe-domain))
(when-let [host (perf/not-empty (:host details))]
(.setHost bq-bldr host))
+ (when-let [^String billing-project-id (perf/not-empty (:billing-project-id details))]
+ ;; Jobs are created and billed in this project; queried tables can live elsewhere.
+ (.setProjectId bq-bldr billing-project-id))
(.. bq-bldr build getService)))
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
index 917ffef9138..191d0f17c95 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
@@ -46,10 +46,13 @@
.getProjectId))
(defn get-project-id
- "Project-id for `details`. Prefers the user-supplied `:project-id`, falls back to the
- one embedded in the service-account credentials."
- [{:keys [project-id] :as details}]
- (or project-id (database-details->credential-project-id details)))
+ "Data project-id for `details`. Fallback: `:project-id`, then `:billing-project-id`, then the
+ SA credentials' project. `:billing-project-id` is included because a service account granted
+ `bigquery.jobs.create` on a project typically also has data access there — so setting only the
+ billing field implies the data lives there too. Users who want billing ≠ data set both keys
+ explicitly."
+ [{:keys [project-id billing-project-id] :as details}]
+ (or project-id billing-project-id (database-details->credential-project-id details)))
(mu/defn populate-project-id-from-credentials!
"Update the given `database` details blob to include the credentials' project-id as a separate entry (under a
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index b9f8ae8e23c..f6e157ea1e0 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -1055,6 +1055,47 @@
(testing " has project-id-from-credentials set correctly"
(is (= (bigquery-project-id) (get-in temp-db [:details :project-id-from-credentials])))))))))
+(deftest billing-project-override-test
+ (mt/test-driver :bigquery-cloud-sdk
+ (testing (str ":billing-project-id sets the BigQuery client's default project "
+ "(which the SDK uses to route query jobs) #73271.")
+ (let [captured (atom nil)
+ orig-fn (mt/original-fn #'bigquery/database-details->client)]
+ (mt/with-dynamic-fn-redefs [bigquery/database-details->client
+ (fn [details]
+ (let [client (orig-fn details)]
+ (reset! captured client)
+ client))]
+ (testing "unset -> client uses the SA's home project (from credentials)"
+ (reset! captured nil)
+ (driver/can-connect? :bigquery-cloud-sdk (:details (mt/db)))
+ (is (= (bigquery-project-id)
+ (.getProjectId (.getOptions ^BigQuery @captured)))))
+ (testing "set -> client uses the billing-project-id"
+ (reset! captured nil)
+ ;; The projectId is not validated at client-build time or by .listDatasets (which takes an
+ ;; explicit projectId). We only need to trigger client construction and inspect it.
+ (driver/can-connect? :bigquery-cloud-sdk
+ (assoc (:details (mt/db))
+ :billing-project-id "no-such-billing-project"))
+ (is (= "no-such-billing-project"
+ (.getProjectId (.getOptions ^BigQuery @captured))))))
+ (testing ":billing-project-id also fills in as data project when :project-id is unset"
+ (let [details (dissoc (:details (mt/db)) :project-id)]
+ (is (= "billing-doubles-as-data"
+ (bigquery.common/get-project-id
+ (assoc details :billing-project-id "billing-doubles-as-data")))
+ "billing-project-id fills in when project-id is unset")
+ (is (= "explicit-data"
+ (bigquery.common/get-project-id
+ (assoc details
+ :project-id "explicit-data"
+ :billing-project-id "different-billing")))
+ ":project-id wins when both are set")
+ (is (= (bigquery-project-id)
+ (bigquery.common/get-project-id details))
+ "SA credentials' project when neither is set (regression)")))))))
+
(deftest bigquery-specific-types-test
(testing "Table with decimal types"
(mt/test-driver