The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Introduce billing-project for BigQuery (#79489)
metambene6216478c671
diff --git a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.tsindex 58077bf2db1..823b241a221 100644--- a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts+++ b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts@@ -32,7 +32,7 @@ const databaseTestCases = [connectionString:"jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=MyBigQueryProject;OAuthType=1;",expectedFields: [- { label: "Project ID (override)", value: "MyBigQueryProject" },+ { label: "Data project ID", value: "MyBigQueryProject" },{ label: "Display name", value: "MyBigQueryProject" },],},diff --git a/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yamlindex 039748659d3..3de0385175b 100644--- a/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml+++ b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml@@ -7,11 +7,17 @@ driver:display-name: BigQuerylazy-load: trueconnection-properties:+ - name: billing-project-id+ display-name: Billing project ID+ helper-text: Google Cloud project where query jobs run and are billed. Defaults to the service account's project.+ placeholder: my-billing-project+ required: false+ type: string- name: project-id- display-name: Project ID (override)- helper-text: Project ID to be used for authentication. You can omit this field if you are only querying datasets owned by your organization.+ display-name: Data project ID+ helper-text: Google Cloud project that owns the datasets you want to query. Defaults to the service account's project.required: false- placeholder: 1w08oDRKPrOqBt06yxY8uiCz2sSvOp3u+ placeholder: my-warehouse-project- name: service-account-jsondisplay-name: Service account JSON filehelper-text: This JSON file contains the credentials Metabase needs to read and query your dataset.diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 78a22764102..dc7631e82f3 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -130,6 +130,9 @@(.setUniverseDomain bq-bldr universe-domain))(when-let [host (perf/not-empty (:host details))](.setHost bq-bldr host))+ (when-let [^String billing-project-id (perf/not-empty (:billing-project-id details))]+ ;; Jobs are created and billed in this project; queried tables can live elsewhere.+ (.setProjectId bq-bldr billing-project-id))(.. bq-bldr build getService)));;; +----------------------------------------------------------------------------------------------------------------+diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.cljindex 917ffef9138..191d0f17c95 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj@@ -46,10 +46,13 @@.getProjectId))(defn get-project-id- "Project-id for `details`. Prefers the user-supplied `:project-id`, falls back to the- one embedded in the service-account credentials."- [{:keys [project-id] :as details}]- (or project-id (database-details->credential-project-id details)))+ "Data project-id for `details`. Fallback: `:project-id`, then `:billing-project-id`, then the+ SA credentials' project. `:billing-project-id` is included because a service account granted+ `bigquery.jobs.create` on a project typically also has data access there — so setting only the+ billing field implies the data lives there too. Users who want billing ≠ data set both keys+ explicitly."+ [{:keys [project-id billing-project-id] :as details}]+ (or project-id billing-project-id (database-details->credential-project-id details)))(mu/defn populate-project-id-from-credentials!"Update the given `database` details blob to include the credentials' project-id as a separate entry (under adiff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex b9f8ae8e23c..f6e157ea1e0 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1055,6 +1055,47 @@(testing " has project-id-from-credentials set correctly"(is (= (bigquery-project-id) (get-in temp-db [:details :project-id-from-credentials])))))))))+(deftest billing-project-override-test+ (mt/test-driver :bigquery-cloud-sdk+ (testing (str ":billing-project-id sets the BigQuery client's default project "+ "(which the SDK uses to route query jobs) #73271.")+ (let [captured (atom nil)+ orig-fn (mt/original-fn #'bigquery/database-details->client)]+ (mt/with-dynamic-fn-redefs [bigquery/database-details->client+ (fn [details]+ (let [client (orig-fn details)]+ (reset! captured client)+ client))]+ (testing "unset -> client uses the SA's home project (from credentials)"+ (reset! captured nil)+ (driver/can-connect? :bigquery-cloud-sdk (:details (mt/db)))+ (is (= (bigquery-project-id)+ (.getProjectId (.getOptions ^BigQuery @captured)))))+ (testing "set -> client uses the billing-project-id"+ (reset! captured nil)+ ;; The projectId is not validated at client-build time or by .listDatasets (which takes an+ ;; explicit projectId). We only need to trigger client construction and inspect it.+ (driver/can-connect? :bigquery-cloud-sdk+ (assoc (:details (mt/db))+ :billing-project-id "no-such-billing-project"))+ (is (= "no-such-billing-project"+ (.getProjectId (.getOptions ^BigQuery @captured))))))+ (testing ":billing-project-id also fills in as data project when :project-id is unset"+ (let [details (dissoc (:details (mt/db)) :project-id)]+ (is (= "billing-doubles-as-data"+ (bigquery.common/get-project-id+ (assoc details :billing-project-id "billing-doubles-as-data")))+ "billing-project-id fills in when project-id is unset")+ (is (= "explicit-data"+ (bigquery.common/get-project-id+ (assoc details+ :project-id "explicit-data"+ :billing-project-id "different-billing")))+ ":project-id wins when both are set")+ (is (= (bigquery-project-id)+ (bigquery.common/get-project-id details))+ "SA credentials' project when neither is set (regression)")))))))+(deftest bigquery-specific-types-test(testing "Table with decimal types"(mt/test-driver