Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix BigQuery tests (#82106)

Ben Grabowe85fa9c446c4
diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
index 8644364430f..47ba58f2a6d 100644
--- a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
@@ -365,9 +365,6 @@
{:dbname "db_router_data"
:enable-multiple-db false})
-(defmethod router-dataset-details :bigquery-cloud-sdk [driver]
- {:dataset-filters-patterns (str "*" (str/replace (router-dataset-name driver) "-" "_"))})
-
(defmethod router-dataset-details :databricks [driver]
{:multi-level-schema false
:schema-filters-patterns (router-dataset-name driver)})
@@ -385,8 +382,7 @@
:enable-multiple-db false})
(defmethod routed-dataset-details :bigquery-cloud-sdk [driver]
- {:service-account-json (tx/db-test-env-var-or-throw driver :service-account-json-routing)
- :dataset-filters-patterns (str "*" (str/replace (routed-dataset-name driver) "-" "_"))})
+ {:service-account-json (tx/db-test-env-var-or-throw driver :service-account-json-routing)})
(defmethod routed-dataset-details :redshift [driver]
{:db (tx/db-test-env-var-or-throw driver :db-routing)})
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index f6e157ea1e0..ef53a0300d1 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -323,6 +323,27 @@
(is (= expected-reqs @requests))
(is (>= expected-size 1000)))))))
+(deftest ^:synchronized later-page-fetch-failure-test
+ (let [row (field-value-list [(prim-cell "x")])
+ schema (Schema/of (u/varargs Field [(Field/of "c0" LegacySQLTypeName/STRING no-fields)]))
+ ;; A non-blank token promises another page, so `adaptive-query-next-page` has to fetch one.
+ first-page (mock-query-page "tok" schema [row])
+ consume #(#'bigquery/bigquery-execute-response
+ first-page nil nil
+ (fn [_cols reducible] (into [] reducible))
+ nil)]
+ (testing "a later page that comes back nil is reported, not silently truncated (#47339)"
+ (with-redefs [bigquery/query-results-page (fn [_job _opts] nil)]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Cannot get next page from BigQuery"
+ (consume)))))
+ (testing "a later page that throws surfaces the original error"
+ (with-redefs [bigquery/query-results-page (fn [_job _opts]
+ (throw (ex-info "onoes BigQuery failed to fetch a later page" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"onoes BigQuery failed to fetch a later page"
+ (consume)))))))
+
;; These look like the macros from metabase.query-processor.expressions-test
;; but conform to bigquery naming rules
(defn- calculate-bird-scarcity* [formula filter-clause]
@@ -1353,44 +1374,18 @@
(is (< count-after (+ count-before 5))
"unbounded thread growth!")))))))
-(deftest later-page-fetch-returns-nil-test
- (mt/test-driver :bigquery-cloud-sdk
- (mt/dataset test-data
- (testing "BigQuery query whose later page fetch returns nil is caught, not silently truncated"
- ;; The query path pages via `query-results-page` (`.getQueryResults`), so simulate BigQuery returning nil
- ;; for a later page even though the page token reported there was more.
- (let [page-counter (atom 3)
- orig-fetch (mt/original-fn #'bigquery/query-results-page)]
- (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]
- (if (zero? @page-counter)
- nil
- (orig-fetch job options)))]
- (binding [bigquery/*page-size* 10 ; small pages so there are several
- bigquery/*page-callback* (fn []
- (let [pages (swap! page-counter #(max (dec %) 0))]
- (log/debugf "*page-callback counting down: %d to go" pages)))]
- (mt/dataset test-data
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Cannot get next page from BigQuery"
- (mt/process-query (mt/query orders))))))))))))
-
(deftest later-page-fetch-throws-test
(mt/test-driver :bigquery-cloud-sdk
- (testing "BigQuery query whose later page fetch throws is caught, with no thread leaks"
- (let [count-before (count (future-thread-names))
- page-counter (atom 3)
- orig-fetch (mt/original-fn #'bigquery/query-results-page)]
- (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]
- (if (zero? @page-counter)
- (throw (ex-info "onoes BigQuery failed to fetch a later page" {}))
- (orig-fetch job options)))]
+ ;; That the error surfaces at all is [[later-page-fetch-failure-test]]'s job, on mock pages. This one runs a real
+ ;; query for what that harness cannot see: the threads `execute-bigquery` starts around the fetch.
+ (testing "BigQuery query whose later page fetch throws leaks no threads"
+ (let [count-before (count (future-thread-names))]
+ ;; `query-results-page` serves only pages *after* the first -- `execute-bigquery` fetches the initial page
+ ;; from `.getQueryResults` itself -- so its first call is already a later page.
+ (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [_job _options]
+ (throw (ex-info "onoes BigQuery failed to fetch a later page" {})))]
(dotimes [_ 10]
- (reset! page-counter 3)
- (binding [bigquery/*page-size* 100 ; small pages so there are several
- bigquery/*page-callback* (fn []
- (let [pages (swap! page-counter #(max (dec %) 0))]
- (log/debugf "*page-callback counting down: %d to go" pages)))]
+ (binding [bigquery/*page-size* 100] ; small first page so there is a second one
(mt/dataset test-data
(is (thrown-with-msg? Exception #"onoes BigQuery failed to fetch a later page"
(mt/process-query (mt/query orders))))))))