Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix BigQuery OOM with information_schema queries (#76459)

Alexander Polyankinf573bc8dbaf7
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index b90410534ec..98609c40288 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -439,7 +439,7 @@
(defn- describe-dataset-table
"Build the field descriptions for a single table from its joined `COLUMNS`/`COLUMN_FIELD_PATHS` rows (see
[[describe-dataset-fields-reducible]]). Each top-level column appears once per nested leaf, or once with a `nil`
- `:field_path` when it has none, so de-dup the columns by `:ordinal_position` and build the nested-field lookup from
+ `:field_path` when it has none, so de-dup the columns by `:column_name` and build the nested-field lookup from
the rows that carry a `:field_path` (whose leaf type is in `:nested_data_type`)."
[dataset-id table-rows]
(let [table-name (:table_name (first table-rows))
@@ -448,7 +448,10 @@
(eduction (filter :field_path)
(map #(assoc % :data_type (:nested_data_type %)))
table-rows))
- columns (into [] (m/distinct-by :ordinal_position) table-rows)]
+ ;; de-dup by `:column_name`, not `:ordinal_position`: BigQuery reports a NULL `ordinal_position` for
+ ;; pseudo-columns (e.g. `_PARTITIONTIME`), and a table can carry more than one, so keying on position would
+ ;; collapse distinct columns into one.
+ columns (into [] (m/distinct-by :column_name) table-rows)]
(sort-by (juxt :table-name :database-position :name)
(describe-dataset-rows nested-lookup dataset-id table-name columns))))
@@ -579,9 +582,14 @@
cap to leave headroom for JVM object expansion when the page is parsed."
(* 4 1024 1024))
-(def ^:private sample-probe-rows
- "Rows to request for the first (probe) page, before we've measured the table's real row size. Small enough to stay
- within the budget even for heavy rows, but not 1 -- a handful averages out per-row size variance."
+(def ^:private initial-page-rows
+ "Rows to request for the *first* result page of every BigQuery fetch -- both `tabledata.list` sampling and regular
+ `getQueryResults` query execution (unless [[*page-size*]] is explicitly set). It's a small probe: the library
+ otherwise requests an unbounded first page, so a wide/large result (e.g. the `INFORMATION_SCHEMA.COLUMNS` sweep in
+ `describe-fields` over a 1000-column dataset, or a heavy sample) materializes hundreds of thousands of `FieldValue`s
+ at once and can OOM sync. After this probe, [[adaptive-sample-next-page]]/[[adaptive-query-next-page]] grow each
+ subsequent page from the *measured* bytes/row toward [[*page-byte-budget*]]. Small enough to stay within budget even
+ for heavy rows, but not 1 -- a handful averages out per-row size variance."
10)
(def ^:private sample-cell-overhead-bytes
@@ -664,7 +672,7 @@
field-idxs (mapv :database_position fields)
all-parsers (get-field-parsers schema)
parsers (mapv all-parsers field-idxs)
- probe (list-sample-page bq-table (min (long sample-probe-rows) table-rows-sample/max-sample-rows) nil)]
+ probe (list-sample-page bq-table (min (long initial-page-rows) table-rows-sample/max-sample-rows) nil)]
(transduce
(comp (take table-rows-sample/max-sample-rows)
(map (partial extract-fingerprint field-idxs parsers)))
@@ -726,8 +734,8 @@
;;; 2. The "lazy" iteration of `TableResult` done by the QP. Any exceptions, or `cancel-chan` checking will be done in the context of the pipeline, solely around the code in `reducible-bigquery-results`.
(def ^:private ^:dynamic ^Long *page-size*
- "Maximum number of rows to return per page in a query. Leave unset (i.e. falling to the library default) by default,
- but override for testing."
+ "Maximum number of rows to return per page in a query. Leave unset (falls back to [[initial-page-rows]] for the first
+ page, then adaptive sizing) by default, but override for testing."
nil)
(defn- throw-invalid-query [e sql parameters]
@@ -896,7 +904,7 @@
(driver-api/the-classloader)
(try
(*page-callback*)
- (let [result-options (if *page-size* [(BigQuery$QueryResultsOption/pageSize *page-size*)] [])
+ (let [result-options [(BigQuery$QueryResultsOption/pageSize (or *page-size* initial-page-rows))]
result (.getQueryResults job (u/varargs BigQuery$QueryResultsOption result-options))]
(if result
(deliver result-promise [:ready result])