The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Make bigquery throw when there is a next page that we cannot fetch (#76709)
Alexander Polyankinf1375bb76495
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 98a10f75005..21c54124cae 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -782,12 +782,20 @@;; realizing more rows as per the maximum result size(.setMaxResults *page-size*))))+(defn- query-results-page+ "Fetch one page of query-job results from `job` with the given `options`. A thin wrapper over `.getQueryResults`+ that exists as a redefable seam so tests can simulate BigQuery returning nil for a later page."+ ^TableResult [^Job job options]+ (.getQueryResults job options))+(defn- adaptive-query-next-page"Adaptive page-advance for query-job results (the regular execution path), mirroring [[adaptive-sample-next-page]]but paging via `getQueryResults` -- the query result's own `.getNextPage` re-uses the original page size and can'tbe re-sized. Measures the just-consumed page's real bytes/row and re-issues the next page with a `pageSize`targeting [[*page-byte-budget*]], so a wide or heavy result fetches fewer rows per page instead of holding a- large parsed page in memory. Returns nil once the result set is exhausted."+ large parsed page in memory. Returns nil once the result set is exhausted; throws if BigQuery reports another+ page is available (non-blank page token) but fails to return it, so we surface the error instead of silently+ truncating the result set."[^Job job](let [budget (long *page-byte-budget*)seen (atom {:bytes 0, :rows 0})]@@ -801,11 +809,12 @@:rows (+ (long (:rows s)) (long page-rows))}))](log/trace "BigQuery: Fetching new page")(*page-callback*)- (.getQueryResults job- (u/varargs BigQuery$QueryResultsOption- [(BigQuery$QueryResultsOption/pageSize- (next-page-size budget bytes rows Long/MAX_VALUE))- (BigQuery$QueryResultsOption/pageToken token)]))))))))+ (or (query-results-page job+ (u/varargs BigQuery$QueryResultsOption+ [(BigQuery$QueryResultsOption/pageSize+ (next-page-size budget bytes rows Long/MAX_VALUE))+ (BigQuery$QueryResultsOption/pageToken token)]))+ (throw (ex-info "Cannot get next page from BigQuery" {})))))))))(defn- reducible-bigquery-results"Reducible over the rows of `page` and its successors. `next-page` is the adaptive page-advance: given thediff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex 7210237c9b0..3c247a3b76e 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1160,64 +1160,49 @@(is (< count-after (+ count-before 5))"unbounded thread growth!")))))))-(comment- ;; this appears to have been broken with the pagination changes in- ;; https://github.com/metabase/metabase/pull/76068/changes- (deftest later-page-fetch-returns-nil-test- (mt/test-driver :bigquery-cloud-sdk- (testing "BigQuery queries which fail on later pages are caught properly"- (let [page-counter (atom 3)- orig-exec (mt/original-fn #'bigquery/reducible-bigquery-results)- wrap-result (fn wrap-result [^TableResult result]- (proxy [TableResult] []- (getSchema [] (.getSchema result))- (getValues [] (.getValues result))- (hasNextPage [] (.hasNextPage result))- (getNextPage []- (if (zero? @page-counter)- nil- (wrap-result (.getNextPage result))))))]- (mt/with-dynamic-fn-redefs [bigquery/reducible-bigquery-results (fn [page & args]- (apply orig-exec (wrap-result page) args))]- (binding [bigquery/*page-size* 10 ; small pages so there are several+(deftest later-page-fetch-returns-nil-test+ (mt/test-driver :bigquery-cloud-sdk+ (testing "BigQuery query whose later page fetch returns nil is caught, not silently truncated"+ ;; The query path pages via `query-results-page` (`.getQueryResults`), so simulate BigQuery returning nil+ ;; for a later page even though the page token reported there was more.+ (let [page-counter (atom 3)+ orig-fetch (mt/original-fn #'bigquery/query-results-page)]+ (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]+ (if (zero? @page-counter)+ nil+ (orig-fetch job options)))]+ (binding [bigquery/*page-size* 10 ; small pages so there are several+ bigquery/*page-callback* (fn []+ (let [pages (swap! page-counter #(max (dec %) 0))]+ (log/debugf "*page-callback counting down: %d to go" pages)))]+ (mt/dataset test-data+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Cannot get next page from BigQuery"+ (mt/process-query (mt/query orders)))))))))))++(deftest later-page-fetch-throws-test+ (mt/test-driver :bigquery-cloud-sdk+ (testing "BigQuery query whose later page fetch throws is caught, with no thread leaks"+ (let [count-before (count (future-thread-names))+ page-counter (atom 3)+ orig-fetch (mt/original-fn #'bigquery/query-results-page)]+ (mt/with-dynamic-fn-redefs [bigquery/query-results-page (fn [job options]+ (if (zero? @page-counter)+ (throw (ex-info "onoes BigQuery failed to fetch a later page" {}))+ (orig-fetch job options)))]+ (dotimes [_ 10]+ (reset! page-counter 3)+ (binding [bigquery/*page-size* 100 ; small pages so there are severalbigquery/*page-callback* (fn [](let [pages (swap! page-counter #(max (dec %) 0))](log/debugf "*page-callback counting down: %d to go" pages)))](mt/dataset test-data- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Cannot get next page from BigQuery"- (mt/process-query (mt/query orders)))))))))))-- (deftest later-page-fetch-throws-test- (mt/test-driver :bigquery-cloud-sdk- (testing "BigQuery queries which fail on later pages are caught properly"- (let [count-before (count (future-thread-names))- page-counter (atom 3)- orig-exec (mt/original-fn #'bigquery/reducible-bigquery-results)- wrap-result (fn wrap-result [^TableResult result]- (proxy [TableResult] []- (getSchema [] (.getSchema result))- (getValues [] (.getValues result))- (hasNextPage [] (.hasNextPage result))- (getNextPage []- (if (zero? @page-counter)- (throw (ex-info "onoes BigQuery failed to fetch a later page" {}))- (wrap-result (.getNextPage result))))))]- (mt/with-dynamic-fn-redefs [bigquery/reducible-bigquery-results (fn [page & args]- (apply orig-exec (wrap-result page) args))]- (dotimes [_ 10]- (reset! page-counter 3)- (binding [bigquery/*page-size* 100 ; small pages so there are several- bigquery/*page-callback* (fn []- (let [pages (swap! page-counter #(max (dec %) 0))]- (log/debugf "*page-callback counting down: %d to go" pages)))]- (mt/dataset test-data- (is (thrown-with-msg? Exception #"onoes BigQuery failed to fetch a later page"- (mt/process-query (mt/query orders))))))))- (testing "no thread leaks"- (let [count-after (count (future-thread-names))]- (is (< count-after (+ count-before 5))))))))))+ (is (thrown-with-msg? Exception #"onoes BigQuery failed to fetch a later page"+ (mt/process-query (mt/query orders))))))))+ (testing "no thread leaks"+ (let [count-after (count (future-thread-names))]+ (is (< count-after (+ count-before 5)))))))))(deftest cancel-page-test(mt/test-driver