Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use unicode regex for bigquery alias (#75123)

Riley Thompson6945a211baab
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj
index 122709e8164..2635add7d0b 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj
@@ -338,6 +338,20 @@
(let [{ws-id :id} (ws/create-workspace! {:name (str "ws-e2e-" run-id)
:creator_id (mt/user->id :crowberto)})]
(try
+ ;; --- Pre-cleanup: delete any stale BigQuery service account whose
+ ;; name collides with the one provisioning will create. In CI the
+ ;; app DB is fresh each run so workspace IDs restart at 1, but the
+ ;; shared GCP project may still hold `mb-ws-1` from a prior run.
+ ;; A stale SA carries outdated IAM bindings; reusing it causes
+ ;; `listDatasets` under impersonation to return empty within the
+ ;; wait window, triggering "cannot find any matching datasets".
+ (when (= admin-driver :bigquery-cloud-sdk)
+ (let [iam ((requiring-resolve 'metabase.driver.bigquery-cloud-sdk.workspace-test-util/iam-client) admin-details)
+ pid ((requiring-resolve 'metabase.driver.bigquery-cloud-sdk.workspace-test-util/project-id) admin-details)]
+ (try
+ ((requiring-resolve 'metabase.driver.bigquery-cloud-sdk.workspace-test-util/delete-sa-direct!)
+ iam pid {:id ws-id})
+ (finally (.close ^java.lang.AutoCloseable iam)))))
;; --- Stage 1: provision via the workspace provisioning entrypoint.
;; Drives the same `init-workspace-isolation!` + `grant-workspace-read-access!`
;; multimethods, but through `provisioning/provision-single!`, which writes
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 845cb2754c4..461daf44eea 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -785,8 +785,8 @@
;; numbers, and underscores, start with a letter or underscore, and be at most 128 characters long.
(let [s (-> (str/trim s)
u/remove-diacritical-marks
- (str/replace #"[^\w\d_]" "_")
- (str/replace #"(^\d)" "_$1"))]
+ (str/replace #"[^\p{L}\p{N}\p{M}\p{Pc}]" "_")
+ (str/replace #"(^[^\p{L}_])" "_$1"))]
((get-method driver/escape-alias :sql) driver s)))
;; See:
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index b42a6f3af92..90da748de0f 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -1033,10 +1033,32 @@
(testing "replace non-letter characters with underscores"
(is (= "_"
(driver/escape-alias :bigquery-cloud-sdk "😍"))))
+ (testing "preserve Unicode letters"
+ (is (= "通過後内定承諾応募数"
+ (driver/escape-alias :bigquery-cloud-sdk "通過後内定承諾応募数"))))
(testing "trim long strings"
(is (= "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_89971909"
(driver/escape-alias :bigquery-cloud-sdk (str/join (repeat 300 "a"))))))))
+(deftest ^:parallel japanese-aggregation-aliases-test
+ (driver/with-driver :bigquery-cloud-sdk
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))
+ price (lib.metadata/field mp (mt/id :products :price))
+ rating (lib.metadata/field mp (mt/id :products :rating))
+ category (lib.metadata/field mp (mt/id :products :category))
+ query (as-> (lib/query mp products) q
+ (lib/aggregate q (lib/with-expression-name (lib/sum price) "通過後内定承諾応募数"))
+ (lib/aggregate q (lib/with-expression-name (lib/sum rating) "通過後入社決定応募数"))
+ (lib/breakout q category)
+ (lib/order-by q (lib/aggregation-ref q 0))
+ (lib/order-by q (lib/aggregation-ref q 1)))]
+ (is (= [["Doohickey" 2185 156]
+ ["Gizmo" 2834 185]
+ ["Gadget" 3019 181]
+ ["Widget" 3109 170]]
+ (mt/formatted-rows [str int int] (qp/process-query query)))))))
+
(deftest ^:parallel remove-diacriticals-from-field-aliases-test
(mt/test-driver :bigquery-cloud-sdk
(testing "We should remove diacriticals and other disallowed characters from field aliases (#14933)"
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
index 6c363c0ef60..59e9de527cd 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
@@ -363,7 +363,7 @@
;;; (#'bq-tx/orphan-isolation-datasets)
;;; ;; => ("mb__isolation_..." ...)
;;; (#'bq-tx/orphan-isolation-service-accounts)
-;;; ;; => ("mb__isolation_..." ...)
+;;; ;; => ("mb-ws-..." ...)
;;;
;;; The whole-orchestration entry point is the existing
;;; [[delete-old-datasets-if-needed!]].
@@ -381,7 +381,7 @@
(map first)))
(defn- orphan-isolation-service-accounts
- "Return iso SA email addresses (`mb__isolation_*@...`) older than 3 hours.
+ "Return iso SA email addresses (`mb-ws-*@...`) older than 3 hours.
Age comes from the `created-at:<iso-instant>` marker encoded in the SA
`description` field by
@@ -403,7 +403,7 @@
(keep (fn [^com.google.iam.admin.v1.ServiceAccount sa]
(let [sa-email (.getEmail sa)
created-at (bigquery.ws/ws-sa-description->created-at (.getDescription sa))]
- (when (and (str/starts-with? sa-email "mb__isolation_")
+ (when (and (str/starts-with? sa-email "mb-ws-")
created-at
(.isBefore ^java.time.Instant created-at threshold))
sa-email))))