Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix OOM with BigQuery and Redshift sync (#76276)

Alexander Polyankinea63bc4889fc
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 84bb5fb2df2..572cf080e8a 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -369,37 +369,31 @@
See https://cloud.google.com/bigquery/docs/querying-partitioned-tables#query_an_ingestion-time_partitioned_table"
"_PARTITIONDATE")
-(defn- get-nested-columns-for-tables
- "Returns nested columns for a specific set of tables"
- [driver database project-id dataset-id table-names]
- (let [results (try (query-honeysql
- driver
- database
- {:select [:table_name :column_name :data_type :field_path]
- :from [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :c]]
- :where [:and
- [:in :table_name table-names]
- ;; we're only interested in nested fields
- [:> [:strpos :field_path "."] 0]]})
- (catch Throwable e
- (log/warnf e "error in get-nested-columns-for-tables for dataset: %s" dataset-id)))
- nested-column-info (fn [{data-type :data_type field-path-str :field_path table-name :table_name}]
- (let [field-path (str/split field-path-str #"\.")
- [database-type base-type] (raw-type->database+base-type data-type)]
- (when-let [nfc-path (not-empty (pop field-path))]
- {:name (peek field-path)
- :table-name table-name
- :table-schema dataset-id
- :database-type database-type
- :base-type base-type
- :nfc-path nfc-path})))]
- (transduce
- (keep nested-column-info)
- (completing
- (fn [accum col]
- (update-in accum [(:table-name col) (:nfc-path col)] (fnil conj []) col)))
- {}
- results)))
+(defn- column-field-path-row->nested-col
+ "Map a `COLUMN_FIELD_PATHS` row to a nested-column map (later keyed by its `:nfc-path`), or `nil` for a top-level
+ (non-nested) field."
+ [dataset-id {data-type :data_type field-path-str :field_path table-name :table_name}]
+ (let [field-path (str/split field-path-str #"\.")
+ [database-type base-type] (raw-type->database+base-type data-type)]
+ (when-let [nfc-path (not-empty (pop field-path))]
+ {:name (peek field-path)
+ :table-name table-name
+ :table-schema dataset-id
+ :database-type database-type
+ :base-type base-type
+ :nfc-path nfc-path})))
+
+(defn- nested-rows->table-lookup
+ "Build a single table's nested-column lookup `{nfc-path [cols]}` from that table's `COLUMN_FIELD_PATHS` rows.
+ Returns `{}` when the table has no nested fields (`table-nested-rows` is empty/`nil`)."
+ [dataset-id table-nested-rows]
+ (transduce
+ (keep #(column-field-path-row->nested-col dataset-id %))
+ (completing
+ (fn [accum col]
+ (update accum (:nfc-path col) (fnil conj []) col)))
+ {}
+ table-nested-rows))
(defn- maybe-add-nested-fields [nested-column-lookup col nfc-path root-database-position]
(let [new-path (conj (or nfc-path []) (:name col))
@@ -443,24 +437,50 @@
table-rows)))
(defn- describe-dataset-fields-reducible
+ "Reducibly describe the fields (including nested STRUCT fields) of `table-names` within `dataset-id`.
+
+ Streams two `INFORMATION_SCHEMA` queries -- `COLUMNS` and `COLUMN_FIELD_PATHS` -- both ordered by `table_name`, and
+ merge-joins them by `table_name`, so we only ever hold ONE table's columns and nested-field paths in memory at a
+ time. Realizing the whole batch's nested-field lookup instead OOMs for wide, deeply-nested datasets (e.g.
+ GA4/Firebase exports: hundreds of daily `events_*` tables, each with hundreds of nested STRUCT leaves). Both queries
+ are single-pass live results, so the returned reducible is single-consumption."
[driver database project-id dataset-id table-names]
(assert (seq table-names))
- (let [named-rows-query {:select [:table_name :column_name :data_type :ordinal_position
- [[:= :is_partitioning_column "YES"] :partitioned]]
- :from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]
- :where [:in :table_name table-names]
- :order-by [:table_name]}
- named-rows (try (query-honeysql driver database named-rows-query)
- (catch Throwable e
- (log/warnf e "error in describe-fields for dataset: %s" dataset-id)))
- nested-column-lookup (get-nested-columns-for-tables driver database project-id dataset-id table-names)]
+ (let [columns-reducible (try (query-honeysql driver database
+ {:select [:table_name :column_name :data_type :ordinal_position
+ [[:= :is_partitioning_column "YES"] :partitioned]]
+ :from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]
+ :where [:in :table_name table-names]
+ :order-by [:table_name]})
+ (catch Throwable e
+ (log/warnf e "error in describe-fields for dataset: %s" dataset-id)))
+ nested-reducible (try (query-honeysql driver database
+ {:select [:table_name :column_name :data_type :field_path]
+ :from [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :c]]
+ :where [:and
+ [:in :table_name table-names]
+ ;; we're only interested in nested fields
+ [:> [:strpos :field_path "."] 0]]
+ :order-by [:table_name]})
+ (catch Throwable e
+ (log/warnf e "error in get-nested-columns-for-tables for dataset: %s" dataset-id)))
+ ;; Lazy per-table groups of `COLUMN_FIELD_PATHS` rows, ordered by `table_name`. We advance through them in
+ ;; lockstep with the per-table `COLUMNS` groups below, keeping only the current table's group realized.
+ group-table (fn [group] (:table_name (first group)))
+ nested-groups (volatile! (partition-by :table_name nested-reducible))
+ lookup-for-table (fn [table-name]
+ (let [remaining (drop-while #(neg? (compare (group-table %) table-name)) @nested-groups)
+ this-table (when (= table-name (group-table (first remaining)))
+ (first remaining))]
+ (vreset! nested-groups (cond-> remaining this-table next))
+ (nested-rows->table-lookup dataset-id this-table)))]
(eduction
(partition-by :table_name)
(mapcat (fn [table-rows]
(let [table-name (:table_name (first table-rows))]
- (->> (describe-dataset-rows (get nested-column-lookup table-name) dataset-id table-name table-rows)
+ (->> (describe-dataset-rows (lookup-for-table table-name) dataset-id table-name table-rows)
(sort-by (juxt :table-name :database-position :name))))))
- named-rows)))
+ columns-reducible)))
;; we redef this in a test, don't make `^:const`!
(def ^:private num-table-partitions
@@ -543,10 +563,10 @@
;;; wildly inaccurate -- a TEXT column may hold 5 bytes or 5 MB), we *measure* it: fetch a small probe page, then
;;; recompute the next page size from the average bytes/row actually seen, so each page targets a fixed byte budget.
-(def ^:private ^:dynamic *sample-page-byte-budget*
- "Target measured bytes per `tabledata.list` sample page. The next page size is `budget / measured-bytes-per-row`,
- clamped to [1, remaining]. Kept well under the server's ~10 MB page cap to leave headroom for JVM object expansion
- when the page is parsed."
+(def ^:private ^:dynamic *page-byte-budget*
+ "Target measured bytes per result page, for both `tabledata.list` sampling and regular query execution. The next
+ page size is `budget / measured-bytes-per-row`, clamped to [1, remaining]. Kept well under the server's ~10 MB page
+ cap to leave headroom for JVM object expansion when the page is parsed."
(* 4 1024 1024))
(def ^:private sample-probe-rows
@@ -577,9 +597,10 @@
^long [^FieldValueList row]
(reduce (fn [^long acc cell] (+ acc (field-value-bytes cell))) 0 row))
-(defn- next-sample-page-size
+(defn- next-page-size
"Rows to request for the next page given the average measured bytes/row so far, targeting `budget` bytes/page.
- Clamped to [1, `remaining`]."
+ Clamped to [1, `remaining`]. Shared by the sampler ([[adaptive-sample-next-page]]) and regular query execution
+ ([[adaptive-query-next-page]])."
^long [^long budget ^long measured-bytes ^long measured-rows ^long remaining]
(let [avg-row-bytes (max 1 (quot measured-bytes (max 1 measured-rows)))]
(-> (quot budget avg-row-bytes)
@@ -595,11 +616,11 @@
(defn- adaptive-sample-next-page
"A swappable page-advance for [[reducible-bigquery-results]], used when sampling. It measures the just-consumed
- page's real bytes/row and re-issues `.list` from the page token with a size targeting `*sample-page-byte-budget*`
+ page's real bytes/row and re-issues `.list` from the page token with a size targeting `*page-byte-budget*`
-- a sliding window that adapts per page to the table's actual data instead of guessing from column types. Returns
nil once the `max-rows` budget is spent or there are no more pages."
[^Table bq-table ^long max-rows]
- (let [budget (long *sample-page-byte-budget*)
+ (let [budget (long *page-byte-budget*)
seen (atom {:bytes 0, :rows 0})]
(fn [^TableResult page]
(let [token (.getNextPageToken page)]
@@ -612,7 +633,7 @@
remaining (- max-rows (long rows))]
(when (pos? remaining)
(*page-callback*)
- (list-sample-page bq-table (next-sample-page-size budget bytes rows remaining) token))))))))
+ (list-sample-page bq-table (next-page-size budget bytes rows remaining) token))))))))
(defn- sample-table
"Process a sample of rows of fields corresponding to the Metabase fields
@@ -743,22 +764,36 @@
;; realizing more rows as per the maximum result size
(.setMaxResults *page-size*))))
-(defn- next-page-same-size
- "Default page-advance for [[reducible-bigquery-results]]: fetch the next page at the original request's page size
- (BigQuery's `.getNextPage`). Returns nil once the result set is exhausted."
- ^TableResult [^TableResult page]
- (when (.hasNextPage page)
- (log/trace "BigQuery: Fetching new page")
- (*page-callback*)
- (or (.getNextPage page)
- (throw (ex-info "Cannot get next page from BigQuery" {})))))
+(defn- adaptive-query-next-page
+ "Adaptive page-advance for query-job results (the regular execution path), mirroring [[adaptive-sample-next-page]]
+ but paging via `getQueryResults` -- the query result's own `.getNextPage` re-uses the original page size and can't
+ be re-sized. Measures the just-consumed page's real bytes/row and re-issues the next page with a `pageSize`
+ targeting [[*page-byte-budget*]], so a wide or heavy result fetches fewer rows per page instead of holding a
+ large parsed page in memory. Returns nil once the result set is exhausted."
+ [^BigQuery client job-id]
+ (let [budget (long *page-byte-budget*)
+ seen (atom {:bytes 0, :rows 0})]
+ (fn [^TableResult page]
+ (let [token (.getNextPageToken page)]
+ (when-not (str/blank? token)
+ (let [[page-bytes page-rows] (reduce (fn [[b n] row] [(+ (long b) (row-bytes row)) (inc (long n))])
+ [0 0]
+ (.getValues page))
+ {:keys [bytes rows]} (swap! seen (fn [s] {:bytes (+ (long (:bytes s)) (long page-bytes))
+ :rows (+ (long (:rows s)) (long page-rows))}))]
+ (log/trace "BigQuery: Fetching new page")
+ (*page-callback*)
+ (.getQueryResults client job-id
+ (u/varargs BigQuery$QueryResultsOption
+ [(BigQuery$QueryResultsOption/pageSize
+ (next-page-size budget bytes rows Long/MAX_VALUE))
+ (BigQuery$QueryResultsOption/pageToken token)]))))))))
(defn- reducible-bigquery-results
- "Reducible over the rows of `page` and its successors. `next-page` is the swappable page-advance: given the
- just-exhausted page it returns the next one (or nil when done). Defaults to [[next-page-same-size]] (fixed page
- size); the sync sampler passes [[adaptive-sample-next-page]] to resize each page from measured bytes/row."
- ([^TableResult page cancel-chan attempt-job-cancel-fn]
- (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn next-page-same-size))
+ "Reducible over the rows of `page` and its successors. `next-page` is the adaptive page-advance: given the
+ just-exhausted page it measures it and returns the next one (re-sized from the measured bytes/row to a byte budget),
+ or nil when done. Adaptive page sizing is the only mode -- callers pass [[adaptive-sample-next-page]] (sampling, via
+ `.list`) or [[adaptive-query-next-page]] (query execution, via `getQueryResults`)."
([^TableResult page cancel-chan attempt-job-cancel-fn next-page]
(reify
clojure.lang.IReduceInit
@@ -825,7 +860,8 @@
cols {:cols columns}
results (eduction (map (fn [^FieldValueList row]
(mapv parse-field-value row parsers)))
- (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn))]
+ (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn
+ (adaptive-query-next-page client job-id)))]
(respond cols results)))
(defn- execute-bigquery
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index a7f89c1dc6a..a20f316288b 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -159,8 +159,8 @@
(is (= (+ (field-value-bytes (prim-cell "aa")) (field-value-bytes (prim-cell "bbb")))
(row-bytes (field-value-list [(prim-cell "aa") (prim-cell "bbb")])))))))
-(deftest ^:parallel next-sample-page-size-test
- (let [next-size @#'bigquery/next-sample-page-size]
+(deftest ^:parallel next-page-size-test
+ (let [next-size @#'bigquery/next-page-size]
(testing "page size = budget / measured-avg-bytes-per-row"
;; 50 rows totalling 500 bytes -> avg 10 -> 1000/10 = 100
(is (= 100 (next-size 1000 500 50 1000000))))
@@ -181,7 +181,7 @@
(deftest ^:parallel reducible-bigquery-results-nil-page-test
(testing "a nil initial page reduces to an empty result instead of NPEing (#47339)"
- (is (= [] (into [] (#'bigquery/reducible-bigquery-results nil nil (constantly nil)))))))
+ (is (= [] (into [] (#'bigquery/reducible-bigquery-results nil nil (constantly nil) (constantly nil)))))))
(deftest ^:synchronized adaptive-sample-next-page-test
(let [requested (atom [])
@@ -190,7 +190,7 @@
(with-redefs [bigquery/list-sample-page (fn [_bq size _token]
(swap! requested conj size)
(mock-page nil []))]
- (binding [bigquery/*sample-page-byte-budget* budget]
+ (binding [bigquery/*page-byte-budget* budget]
((#'bigquery/adaptive-sample-next-page :table max-rows) (mock-page "tok" rows))
(first @requested))))
light (vec (repeatedly 5 #(field-value-list [(prim-cell "x")])))
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index f82f31dd9c6..1bda8877981 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -93,16 +93,13 @@
(defmethod sql-jdbc.sync/describe-fields-pre-process-xf :redshift
[_driver _db & _args]
- (fn [rf]
- (let [fields (volatile! (transient []))]
- (fn
- ([] (rf))
- ([result]
- (let [filtered (remove-duplicate-fields (persistent! @fields))]
- (rf (reduce rf result filtered))))
- ([result field]
- (vswap! fields conj! field)
- result)))))
+ ;; `describe-fields-sql` orders by [table-schema table-name database-position], so each table's columns arrive
+ ;; contiguously. A duplicate-column key is (table-schema, table-name, name) -- by definition all its occurrences are
+ ;; within a single table -- so we can dedup per table with `partition-by` rather than buffering the entire result
+ ;; set. This bounds memory to one table's columns (the per-table streaming contract) and is otherwise identical to a
+ ;; global dedup.
+ (comp (partition-by (juxt :table-schema :table-name))
+ (mapcat remove-duplicate-fields)))
;; Skip the postgres implementation as it has to handle custom enums which redshift doesn't support.
(defmethod driver/dynamic-database-types-lookup :redshift