The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Check via .getDataset instead of .listDatasets when all datasets are exactly named (#79921)
Ben Grabowa0d8a1758b59
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 7b162d29e59..ab3667898e5 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -29,7 +29,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [mapv some empty? not-empty]]+ [metabase.util.performance :refer [every? mapv some empty? not-empty]]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2])(:import@@ -143,22 +143,74 @@;;; | Sync |;;; +----------------------------------------------------------------------------------------------------------------++(def ^:private exact-dataset-id-re+ "Matches an inclusion-filter segment that names one dataset outright.++ [[metabase.driver.sync/schema-pattern->re-pattern]] compiles a segment into a regex, expanding an unescaped `*`+ into `.*` and passing every other character through as regex source. A segment of only `\\w` characters therefore+ contains no wildcard and no regex syntax, so it matches itself and nothing else -- and `\\w` is also exactly the+ character set BigQuery permits in a dataset ID. A leading underscore is excluded on top of that: BigQuery calls such+ a dataset *hidden* and omits it from `datasets.list` unless `all=true`, which we never pass, so a scan would never+ turn one up even though a name lookup finds it.++ All of that has to hold for a name lookup to select the same datasets a scan would;+ [[metabase.driver.bigquery-cloud-sdk-test/exactly-named-datasets-agrees-with-scan-test]] pins that agreement."+ #"[a-zA-Z0-9]\w*")++(defn- exactly-named-datasets+ "The dataset IDs an inclusion filter names outright, or `nil` when evaluating the filter requires a scan.++ Blank patterns mean \"include everything\", so they fall through to the scan."+ [{:keys [dataset-filters-type dataset-filters-patterns]}]+ (when (= "inclusion" dataset-filters-type)+ (let [segments (map str/trim (str/split (str dataset-filters-patterns) #","))]+ (when (every? #(re-matches exact-dataset-id-re %) segments)+ ;; a scan yields each dataset once however many segments matched it+ (distinct segments)))))++(defn- dataset-exists?+ [^BigQuery client project-id dataset-id]+ (some? (.getDataset client (DatasetId/of project-id dataset-id) (u/varargs BigQuery$DatasetOption))))++(defn- assert-project-reachable!+ "Throws whatever listing would have thrown when `project-id` is unreachable.++ `.getDataset` answers 404 with `nil`, so a name lookup that comes back empty cannot tell \"no such dataset\" from+ \"no such project\" -- and callers rely on the difference: [[driver/can-connect?]] reports a bad project as a+ connection failure, and sync must not read an unreachable project as one holding no tables. `.listDatasets` throws+ in that case, and a single-row page is enough to provoke it."+ [^BigQuery client project-id]+ (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption [(BigQuery$DatasetListOption/pageSize 1)]))+ nil)+(defn- list-datasets- "Fetch all datasets given database `details`, applying dataset filters if specified."+ "Fetch all datasets given database `details`, applying dataset filters if specified.++ When the filter names its datasets outright they are fetched by name. Listing is paginated over every dataset in+ the project, which is unrelated to how many the filter keeps -- 3400 datasets took ~18s, past the+ [[metabase.driver.settings/db-connection-timeout-ms]] budget that `can-connect?` runs under."[{:keys [dataset-filters-type dataset-filters-patterns] :as details} & {:keys [logging-schema-exclusions?]}](let [client (database-details->client details)- project-id (bigquery.common/get-project-id details)- datasets (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption))- inclusion-patterns (when (= "inclusion" dataset-filters-type) dataset-filters-patterns)- exclusion-patterns (when (= "exclusion" dataset-filters-type) dataset-filters-patterns)]- (for [^Dataset dataset (.iterateAll datasets)- :let [dataset-id (.. dataset getDatasetId getDataset)]- :when ((if logging-schema-exclusions?- sql-jdbc.describe-database/include-schema-logging-exclusion- driver.s/include-schema?) inclusion-patterns- exclusion-patterns- dataset-id)]- dataset-id)))+ project-id (bigquery.common/get-project-id details)]+ (if-let [named (exactly-named-datasets details)]+ ;; `logging-schema-exclusions?` has nothing to report on this path: it logs datasets the filter rejected, and+ ;; here every dataset looked at was named by the filter. Stays lazy past the first hit so `can-connect?` costs+ ;; one round trip.+ (let [found (seq (filter #(dataset-exists? client project-id %) named))]+ (when-not found+ (assert-project-reachable! client project-id))+ found)+ (let [datasets (.listDatasets client project-id (u/varargs BigQuery$DatasetListOption))+ inclusion-patterns (when (= "inclusion" dataset-filters-type) dataset-filters-patterns)+ exclusion-patterns (when (= "exclusion" dataset-filters-type) dataset-filters-patterns)]+ (for [^Dataset dataset (.iterateAll datasets)+ :let [dataset-id (.. dataset getDatasetId getDataset)]+ :when ((if logging-schema-exclusions?+ sql-jdbc.describe-database/include-schema-logging-exclusion+ driver.s/include-schema?) inclusion-patterns+ exclusion-patterns+ dataset-id)]+ dataset-id)))))(defmethod driver/can-connect? :bigquery-cloud-sdk[_ details]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex e138d7836b6..75ee76eb8ca 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -13,6 +13,7 @@[metabase.driver.bigquery-cloud-sdk.common :as bigquery.common][metabase.driver.common.table-rows-sample :as table-rows-sample][metabase.driver.settings :as driver.settings]+ [metabase.driver.sync :as driver.s][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor.compile :as qp.compile]@@ -52,6 +53,42 @@[table-name](bigquery.tx/execute! (format "DROP TABLE IF EXISTS `%s`;" (fmt-table-name table-name))))+(deftest ^:parallel exactly-named-datasets-agrees-with-scan-test+ (testing "a filter of plain names selects exactly the datasets a scan of the whole project would keep"+ (let [universe ["orders" "orders_v2" "ORDERS" "public" "public_archive" "a_b" "x1"]]+ (doseq [patterns ["orders" "ORDERS" "orders,public" " orders , public " "a_b" "x1,orders_v2"+ ;; naming something absent is fine -- it just selects nothing+ "not_a_dataset" "orders,not_a_dataset"]]+ (testing (pr-str patterns)+ (let [named (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"+ :dataset-filters-patterns patterns})]+ (is (some? named)+ "should be recognized as naming its datasets outright")+ ;; only names that exist can come back from a lookup, so compare within the universe+ (is (= (set (filter #(driver.s/include-schema? patterns nil %) universe))+ (set (filter (set universe) named)))))))))+ (testing "a dataset named twice is looked up once, as a scan would yield it once"+ (is (= ["orders" "public"]+ (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"+ :dataset-filters-patterns "orders,public,orders"}))))+ (testing "filters that a name lookup cannot answer fall through to a scan"+ (doseq [[patterns why] {"orders*" "wildcard"+ "*" "wildcard"+ "a,b*" "wildcard in one segment"+ "crazy\\*schema" "escaped asterisk is not a legal dataset ID"+ "_hidden" "leading underscore means hidden; a scan never lists it"+ "orders,_hidden" "one hidden name is enough to need a scan"+ "" "blank means include everything"+ nil "blank means include everything"}]+ (testing why+ (is (nil? (#'bigquery/exactly-named-datasets {:dataset-filters-type "inclusion"+ :dataset-filters-patterns patterns}))))))+ (testing "only inclusion filters name datasets; anything else needs a scan"+ (doseq [filters-type ["exclusion" "all" nil]]+ (testing (pr-str filters-type)+ (is (nil? (#'bigquery/exactly-named-datasets {:dataset-filters-type filters-type+ :dataset-filters-patterns "orders"})))))))+(deftest ^:parallel sanity-check-test(mt/test-driver:bigquery-cloud-sdk