Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Check via .getDataset instead of .listDatasets when all datasets are exactly named (#79921)

Ben Grabowa0d8a1758b59
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 7b162d29e59..ab3667898e5 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -29,7 +29,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [mapv some empty? not-empty]]
+ [metabase.util.performance :refer [every? mapv some empty? not-empty]]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2])
(:import
@@ -143,22 +143,74 @@
;;; | Sync |
;;; +----------------------------------------------------------------------------------------------------------------+
+(def ^:private exact-dataset-id-re
+ "Matches an inclusion-filter segment that names one dataset outright.
+
+ [[metabase.driver.sync/schema-pattern->re-pattern]] compiles a segment into a regex, expanding an unescaped `*`
+ into `.*` and passing every other character through as regex source. A segment of only `\\w` characters therefore
+ contains no wildcard and no regex syntax, so it matches itself and nothing else -- and `\\w` is also exactly the
+ character set BigQuery permits in a dataset ID. A leading underscore is excluded on top of that: BigQuery calls such
+ a dataset *hidden* and omits it from `datasets.list` unless `all=true`, which we never pass, so a scan would never
+ turn one up even though a name lookup finds it.
+
+ All of that has to hold for a name lookup to select the same datasets a scan would;
+ [[metabase.driver.bigquery-cloud-sdk-test/exactly-named-datasets-agrees-with-scan-test]] pins that agreement."
+ #"[a-zA-Z0-9]\w*")
+
+(defn- exactly-named-datasets
+ "The dataset IDs an inclusion filter names outright, or `nil` when evaluating the filter requires a scan.
+
+ Blank patterns mean \"include everything\", so they fall through to the scan."
+ [{:keys [dataset-filters-type dataset-filters-patterns]}]
+ (when (= "inclusion" dataset-filters-type)
+ (let [segments (map str/trim (str/split (str dataset-filters-patterns) #","))]
+ (when (every? #(re-matches exact-dataset-id-re %) segments)
+ ;; a scan yields each dataset once however many segments matched it
+ (distinct segments)))))
+
+(defn- dataset-exists?
+ [^BigQuery client project-id dataset-id]
+ (some? (.getDataset client (DatasetId/of project-id dataset-id) (u/varargs BigQuery$DatasetOption))))
+
+(defn- assert-project-reachable!
+ "Throws whatever listing would have thrown when `project-id` is unreachable.
+
+ `.getDataset` answers 404 with `nil`, so a name lookup that comes back empty cannot tell \"no such dataset\" from
+ \"no such project\" -- and callers rely on the difference: [[driver/can-connect?]] reports a bad project as a
+ connection failure, and sync must not read an unreachable project as one holding no tables. `.listDatasets` throws
+ in that case, and a single-row page is enough to provoke it."
+ [^BigQuery client project-id]
+ (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption [(BigQuery$DatasetListOption/pageSize 1)]))
+ nil)
+
(defn- list-datasets
- "Fetch all datasets given database `details`, applying dataset filters if specified."
+ "Fetch all datasets given database `details`, applying dataset filters if specified.
+
+ When the filter names its datasets outright they are fetched by name. Listing is paginated over every dataset in
+ the project, which is unrelated to how many the filter keeps -- 3400 datasets took ~18s, past the
+ [[metabase.driver.settings/db-connection-timeout-ms]] budget that `can-connect?` runs under."
[{:keys [dataset-filters-type dataset-filters-patterns] :as details} & {:keys [logging-schema-exclusions?]}]
(let [client (database-details->client details)
- project-id (bigquery.common/get-project-id details)
- datasets (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption))
- inclusion-patterns (when (= "inclusion" dataset-filters-type) dataset-filters-patterns)
- exclusion-patterns (when (= "exclusion" dataset-filters-type) dataset-filters-patterns)]
- (for [^Dataset dataset (.iterateAll datasets)
- :let [dataset-id (.. dataset getDatasetId getDataset)]
- :when ((if logging-schema-exclusions?
- sql-jdbc.describe-database/include-schema-logging-exclusion
- driver.s/include-schema?) inclusion-patterns
- exclusion-patterns
- dataset-id)]
- dataset-id)))
+ project-id (bigquery.common/get-project-id details)]
+ (if-let [named (exactly-named-datasets details)]
+ ;; `logging-schema-exclusions?` has nothing to report on this path: it logs datasets the filter rejected, and
+ ;; here every dataset looked at was named by the filter. Stays lazy past the first hit so `can-connect?` costs
+ ;; one round trip.
+ (let [found (seq (filter #(dataset-exists? client project-id %) named))]
+ (when-not found
+ (assert-project-reachable! client project-id))
+ found)
+ (let [datasets (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption))
+ inclusion-patterns (when (= "inclusion" dataset-filters-type) dataset-filters-patterns)
+ exclusion-patterns (when (= "exclusion" dataset-filters-type) dataset-filters-patterns)]
+ (for [^Dataset dataset (.iterateAll datasets)
+ :let [dataset-id (.. dataset getDatasetId getDataset)]
+ :when ((if logging-schema-exclusions?
+ sql-jdbc.describe-database/include-schema-logging-exclusion
+ driver.s/include-schema?) inclusion-patterns
+ exclusion-patterns
+ dataset-id)]
+ dataset-id)))))
(defmethod driver/can-connect? :bigquery-cloud-sdk
[_ details]
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index e138d7836b6..75ee76eb8ca 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -13,6 +13,7 @@
[metabase.driver.bigquery-cloud-sdk.common :as bigquery.common]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sync :as driver.s]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.compile :as qp.compile]
@@ -52,6 +53,42 @@
[table-name]
(bigquery.tx/execute! (format "DROP TABLE IF EXISTS `%s`;" (fmt-table-name table-name))))
+(deftest ^:parallel exactly-named-datasets-agrees-with-scan-test
+ (testing "a filter of plain names selects exactly the datasets a scan of the whole project would keep"
+ (let [universe ["orders" "orders_v2" "ORDERS" "public" "public_archive" "a_b" "x1"]]
+ (doseq [patterns ["orders" "ORDERS" "orders,public" " orders , public " "a_b" "x1,orders_v2"
+ ;; naming something absent is fine -- it just selects nothing
+ "not_a_dataset" "orders,not_a_dataset"]]
+ (testing (pr-str patterns)
+ (let [named (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"
+ :dataset-filters-patterns patterns})]
+ (is (some? named)
+ "should be recognized as naming its datasets outright")
+ ;; only names that exist can come back from a lookup, so compare within the universe
+ (is (= (set (filter #(driver.s/include-schema? patterns nil %) universe))
+ (set (filter (set universe) named)))))))))
+ (testing "a dataset named twice is looked up once, as a scan would yield it once"
+ (is (= ["orders" "public"]
+ (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"
+ :dataset-filters-patterns "orders,public,orders"}))))
+ (testing "filters that a name lookup cannot answer fall through to a scan"
+ (doseq [[patterns why] {"orders*" "wildcard"
+ "*" "wildcard"
+ "a,b*" "wildcard in one segment"
+ "crazy\\*schema" "escaped asterisk is not a legal dataset ID"
+ "_hidden" "leading underscore means hidden; a scan never lists it"
+ "orders,_hidden" "one hidden name is enough to need a scan"
+ "" "blank means include everything"
+ nil "blank means include everything"}]
+ (testing why
+ (is (nil? (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"
+ :dataset-filters-patterns patterns}))))))
+ (testing "only inclusion filters name datasets; anything else needs a scan"
+ (doseq [filters-type ["exclusion" "all" nil]]
+ (testing (pr-str filters-type)
+ (is (nil? (#'bigquery/exactly-named-datasets {:dataset-filters-type filters-type
+ :dataset-filters-patterns "orders"})))))))
+
(deftest ^:parallel sanity-check-test
(mt/test-driver
:bigquery-cloud-sdk