Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use separate page-byte-budget for bigquery query execution and sync (#80849)

Riley Thompson44b811bdccdc
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index dbcedf1f27e..0ab1b61027b 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -641,19 +641,28 @@
;;; recompute the next page size from the average bytes/row actually seen, so each page targets a fixed byte budget.
(def ^:private ^:dynamic *page-byte-budget*
- "Target measured bytes per result page, for both `tabledata.list` sampling and regular query execution. The next
- page size is `budget / measured-bytes-per-row`, clamped to [1, remaining]. Kept well under the server's ~10 MB page
- cap to leave headroom for JVM object expansion when the page is parsed."
+ "Target measured bytes per result page for `tabledata.list` sampling. The next page size is
+ `budget / measured-bytes-per-row`, clamped to [1, remaining]. Kept well under the server's ~10 MB page cap to leave
+ headroom for JVM object expansion when the page is parsed. Sync may have several tables in flight at once, so this
+ stays small. Regular query execution uses the larger [[*query-page-byte-budget*]]."
(* 4 1024 1024))
+(def ^:private ^:dynamic *query-page-byte-budget*
+ "Target measured bytes per result page for regular query execution. Deliberately larger than [[*page-byte-budget*]]
+ because the QP streams rows and holds only one parsed page at a time, and the number of `getQueryResults` round trips
+ is inversely proportional to this budget, so a small budget makes CSV downloads of larger tables >2x slower (#79273).
+ Still a hard bound, so the unbounded-first-page OOM this pagination was added to prevent (#76459) cannot recur."
+ (* 32 1024 1024))
+
(def ^:private initial-page-rows
"Rows to request for the *first* result page of every BigQuery fetch -- both `tabledata.list` sampling and regular
`getQueryResults` query execution (unless [[*page-size*]] is explicitly set). It's a small probe: the library
otherwise requests an unbounded first page, so a wide/large result (e.g. the `INFORMATION_SCHEMA.COLUMNS` sweep in
`describe-fields` over a 1000-column dataset, or a heavy sample) materializes hundreds of thousands of `FieldValue`s
at once and can OOM sync. After this probe, [[adaptive-sample-next-page]]/[[adaptive-query-next-page]] grow each
- subsequent page from the *measured* bytes/row toward [[*page-byte-budget*]]. Small enough to stay within budget even
- for heavy rows, but not 1 -- a handful averages out per-row size variance."
+ subsequent page from the *measured* bytes/row toward that path's byte budget ([[*page-byte-budget*]] for sampling,
+ [[*query-page-byte-budget*]] for query execution). Small enough to stay within budget even for heavy rows, but not
+ 1 -- a handful averages out per-row size variance."
10)
(def ^:private sample-cell-overhead-bytes
@@ -876,23 +885,30 @@
^TableResult [^Job job options]
(.getQueryResults job options))
+(def ^:private measured-rows-per-page
+ "Rows of each consumed page [[adaptive-query-next-page]] measured to estimate bytes/row. Measuring the whole page
+ re-walks every cell a second time after normal result parsing but a prefix estimates the average just as well.
+ Only the query path can take this shortcut: [[adaptive-sample-next-page]] also needs the *exact* per-page row count
+ for its `max-rows` budget, so it still measures every row."
+ 32)
+
(defn- adaptive-query-next-page
"Adaptive page-advance for query-job results (the regular execution path), mirroring [[adaptive-sample-next-page]]
but paging via `getQueryResults` -- the query result's own `.getNextPage` re-uses the original page size and can't
- be re-sized. Measures the just-consumed page's real bytes/row and re-issues the next page with a `pageSize`
- targeting [[*page-byte-budget*]], so a wide or heavy result fetches fewer rows per page instead of holding a
- large parsed page in memory. Returns nil once the result set is exhausted; throws if BigQuery reports another
- page is available (non-blank page token) but fails to return it, so we surface the error instead of silently
- truncating the result set."
+ be re-sized. Measures the just-consumed page's real bytes/row (a [[measured-rows-per-page]] prefix) and re-issues
+ the next page with a `pageSize` targeting [[*query-page-byte-budget*]], so a wide or heavy result fetches fewer
+ rows per page instead of holding a large parsed page in memory. Returns nil once the result set is exhausted;
+ throws if BigQuery reports another page is available (non-blank page token) but fails to return it, so we surface
+ the error instead of silently truncating the result set."
[^Job job]
- (let [budget (long *page-byte-budget*)
+ (let [budget (long *query-page-byte-budget*)
seen (atom {:bytes 0, :rows 0})]
(fn [^TableResult page]
(let [token (.getNextPageToken page)]
(when-not (str/blank? token)
(let [[page-bytes page-rows] (reduce (fn [[b n] row] [(+ (long b) (row-bytes row)) (inc (long n))])
[0 0]
- (.getValues page))
+ (eduction (take measured-rows-per-page) (.getValues page)))
{:keys [bytes rows]} (swap! seen (fn [s] {:bytes (+ (long (:bytes s)) (long page-bytes))
:rows (+ (long (:rows s)) (long page-rows))}))]
(log/trace "BigQuery: Fetching new page")
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index 6d66a83c810..3f5bc1c4488 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -34,7 +34,7 @@
[metabase.warehouse-schema.models.field-values :as field-values]
[toucan2.core :as t2])
(:import
- (com.google.cloud.bigquery BigQuery BigQueryException Field FieldValue FieldValue$Attribute FieldValueList TableResult)
+ (com.google.cloud.bigquery BigQuery BigQueryException Field FieldValue FieldValue$Attribute FieldValueList JobId LegacySQLTypeName Schema TableResult)
(com.google.cloud.http HttpTransportOptions)))
(set! *warn-on-reflection* true)
@@ -277,6 +277,51 @@
(testing "no further page is fetched once the page token is blank"
(is (nil? ((#'bigquery/adaptive-sample-next-page :table 100) (mock-page "" light)))))))
+(defn- mock-query-page
+ "Like [[mock-page]], but for the query-execution path: also exposes the schema and job-id that
+ `bigquery-execute-response` reads off the first page."
+ ^TableResult [token schema rows]
+ (proxy [TableResult] []
+ (getNextPageToken [] token)
+ (getValues [] rows)
+ (getSchema [] schema)
+ (getJobId [] (JobId/of "test-project" "test-job"))))
+
+(deftest ^:synchronized adaptive-query-paging-request-count-test
+ (testing "10,000 rows × 250 cols: pages stay large and round trips stay few (#79273)"
+ (let [n-rows 10000
+ wide-row (field-value-list (repeat 250 (prim-cell "0123456789")))
+ schema (Schema/of (u/varargs Field (for [i (range 250)]
+ (Field/of (str "c" i) LegacySQLTypeName/STRING no-fields))))
+ ;; token is non-blank iff rows remain after this page
+ page (fn [remaining-after rows]
+ (mock-query-page (if (pos? remaining-after) "tok" "") schema rows))
+ remaining (atom (- n-rows @#'bigquery/initial-page-rows))
+ sizes (atom [])
+ requests (atom 0)
+ orig-next-page-size @#'bigquery/next-page-size]
+ (with-redefs [bigquery/next-page-size (fn ^long [^long budget ^long bytes ^long rows ^long rem]
+ (let [n (long (orig-next-page-size budget bytes rows rem))]
+ (swap! sizes conj n)
+ n))
+ bigquery/query-results-page (fn [_job _opts]
+ (swap! requests inc)
+ (let [k (min (long (peek @sizes)) @remaining)
+ rem (swap! remaining - k)]
+ (page rem (vec (repeat k wide-row)))))]
+ (let [{:keys [rows]} (#'bigquery/bigquery-execute-response
+ (page (- n-rows 10) (vec (repeat 10 wide-row)))
+ nil nil
+ (fn [cols reducible] {:cols cols, :rows (into [] reducible)})
+ nil)
+ expected-size (quot (long @#'bigquery/*query-page-byte-budget*)
+ (#'bigquery/row-bytes wide-row))
+ expected-reqs (long (Math/ceil (/ (- n-rows 10) (double expected-size))))]
+ (is (= n-rows (count rows)))
+ (is (every? #(= expected-size %) @sizes))
+ (is (= expected-reqs @requests))
+ (is (>= expected-size 1000)))))))
+
;; These look like the macros from metabase.query-processor.expressions-test
;; but conform to bigquery naming rules
(defn- calculate-bird-scarcity* [formula filter-clause]