The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix OOM with BigQuery and Redshift sync (#76276)
Alexander Polyankinea63bc4889fc
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 84bb5fb2df2..572cf080e8a 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -369,37 +369,31 @@See https://cloud.google.com/bigquery/docs/querying-partitioned-tables#query_an_ingestion-time_partitioned_table""_PARTITIONDATE")-(defn- get-nested-columns-for-tables- "Returns nested columns for a specific set of tables"- [driver database project-id dataset-id table-names]- (let [results (try (query-honeysql- driver- database- {:select [:table_name :column_name :data_type :field_path]- :from [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :c]]- :where [:and- [:in :table_name table-names]- ;; we're only interested in nested fields- [:> [:strpos :field_path "."] 0]]})- (catch Throwable e- (log/warnf e "error in get-nested-columns-for-tables for dataset: %s" dataset-id)))- nested-column-info (fn [{data-type :data_type field-path-str :field_path table-name :table_name}]- (let [field-path (str/split field-path-str #"\.")- [database-type base-type] (raw-type->database+base-type data-type)]- (when-let [nfc-path (not-empty (pop field-path))]- {:name (peek field-path)- :table-name table-name- :table-schema dataset-id- :database-type database-type- :base-type base-type- :nfc-path nfc-path})))]- (transduce- (keep nested-column-info)- (completing- (fn [accum col]- (update-in accum [(:table-name col) (:nfc-path col)] (fnil conj []) col)))- {}- results)))+(defn- column-field-path-row->nested-col+ "Map a `COLUMN_FIELD_PATHS` row to a nested-column map (later keyed by its `:nfc-path`), or `nil` for a top-level+ (non-nested) field."+ [dataset-id {data-type :data_type field-path-str :field_path table-name :table_name}]+ (let [field-path (str/split field-path-str #"\.")+ [database-type base-type] (raw-type->database+base-type data-type)]+ (when-let [nfc-path (not-empty (pop field-path))]+ {:name (peek field-path)+ :table-name table-name+ :table-schema dataset-id+ :database-type database-type+ :base-type base-type+ :nfc-path nfc-path})))++(defn- nested-rows->table-lookup+ "Build a single table's nested-column lookup `{nfc-path [cols]}` from that table's `COLUMN_FIELD_PATHS` rows.+ Returns `{}` when the table has no nested fields (`table-nested-rows` is empty/`nil`)."+ [dataset-id table-nested-rows]+ (transduce+ (keep #(column-field-path-row->nested-col dataset-id %))+ (completing+ (fn [accum col]+ (update accum (:nfc-path col) (fnil conj []) col)))+ {}+ table-nested-rows))(defn- maybe-add-nested-fields [nested-column-lookup col nfc-path root-database-position](let [new-path (conj (or nfc-path []) (:name col))@@ -443,24 +437,50 @@table-rows)))(defn- describe-dataset-fields-reducible+ "Reducibly describe the fields (including nested STRUCT fields) of `table-names` within `dataset-id`.++ Streams two `INFORMATION_SCHEMA` queries -- `COLUMNS` and `COLUMN_FIELD_PATHS` -- both ordered by `table_name`, and+ merge-joins them by `table_name`, so we only ever hold ONE table's columns and nested-field paths in memory at a+ time. Realizing the whole batch's nested-field lookup instead OOMs for wide, deeply-nested datasets (e.g.+ GA4/Firebase exports: hundreds of daily `events_*` tables, each with hundreds of nested STRUCT leaves). Both queries+ are single-pass live results, so the returned reducible is single-consumption."[driver database project-id dataset-id table-names](assert (seq table-names))- (let [named-rows-query {:select [:table_name :column_name :data_type :ordinal_position- [[:= :is_partitioning_column "YES"] :partitioned]]- :from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]- :where [:in :table_name table-names]- :order-by [:table_name]}- named-rows (try (query-honeysql driver database named-rows-query)- (catch Throwable e- (log/warnf e "error in describe-fields for dataset: %s" dataset-id)))- nested-column-lookup (get-nested-columns-for-tables driver database project-id dataset-id table-names)]+ (let [columns-reducible (try (query-honeysql driver database+ {:select [:table_name :column_name :data_type :ordinal_position+ [[:= :is_partitioning_column "YES"] :partitioned]]+ :from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]+ :where [:in :table_name table-names]+ :order-by [:table_name]})+ (catch Throwable e+ (log/warnf e "error in describe-fields for dataset: %s" dataset-id)))+ nested-reducible (try (query-honeysql driver database+ {:select [:table_name :column_name :data_type :field_path]+ :from [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :c]]+ :where [:and+ [:in :table_name table-names]+ ;; we're only interested in nested fields+ [:> [:strpos :field_path "."] 0]]+ :order-by [:table_name]})+ (catch Throwable e+ (log/warnf e "error in get-nested-columns-for-tables for dataset: %s" dataset-id)))+ ;; Lazy per-table groups of `COLUMN_FIELD_PATHS` rows, ordered by `table_name`. We advance through them in+ ;; lockstep with the per-table `COLUMNS` groups below, keeping only the current table's group realized.+ group-table (fn [group] (:table_name (first group)))+ nested-groups (volatile! (partition-by :table_name nested-reducible))+ lookup-for-table (fn [table-name]+ (let [remaining (drop-while #(neg? (compare (group-table %) table-name)) @nested-groups)+ this-table (when (= table-name (group-table (first remaining)))+ (first remaining))]+ (vreset! nested-groups (cond-> remaining this-table next))+ (nested-rows->table-lookup dataset-id this-table)))](eduction(partition-by :table_name)(mapcat (fn [table-rows](let [table-name (:table_name (first table-rows))]- (->> (describe-dataset-rows (get nested-column-lookup table-name) dataset-id table-name table-rows)+ (->> (describe-dataset-rows (lookup-for-table table-name) dataset-id table-name table-rows)(sort-by (juxt :table-name :database-position :name))))))- named-rows)))+ columns-reducible)));; we redef this in a test, don't make `^:const`!(def ^:private num-table-partitions@@ -543,10 +563,10 @@;;; wildly inaccurate -- a TEXT column may hold 5 bytes or 5 MB), we *measure* it: fetch a small probe page, then;;; recompute the next page size from the average bytes/row actually seen, so each page targets a fixed byte budget.-(def ^:private ^:dynamic *sample-page-byte-budget*- "Target measured bytes per `tabledata.list` sample page. The next page size is `budget / measured-bytes-per-row`,- clamped to [1, remaining]. Kept well under the server's ~10 MB page cap to leave headroom for JVM object expansion- when the page is parsed."+(def ^:private ^:dynamic *page-byte-budget*+ "Target measured bytes per result page, for both `tabledata.list` sampling and regular query execution. The next+ page size is `budget / measured-bytes-per-row`, clamped to [1, remaining]. Kept well under the server's ~10 MB page+ cap to leave headroom for JVM object expansion when the page is parsed."(* 4 1024 1024))(def ^:private sample-probe-rows@@ -577,9 +597,10 @@^long [^FieldValueList row](reduce (fn [^long acc cell] (+ acc (field-value-bytes cell))) 0 row))-(defn- next-sample-page-size+(defn- next-page-size"Rows to request for the next page given the average measured bytes/row so far, targeting `budget` bytes/page.- Clamped to [1, `remaining`]."+ Clamped to [1, `remaining`]. Shared by the sampler ([[adaptive-sample-next-page]]) and regular query execution+ ([[adaptive-query-next-page]])."^long [^long budget ^long measured-bytes ^long measured-rows ^long remaining](let [avg-row-bytes (max 1 (quot measured-bytes (max 1 measured-rows)))](-> (quot budget avg-row-bytes)@@ -595,11 +616,11 @@(defn- adaptive-sample-next-page"A swappable page-advance for [[reducible-bigquery-results]], used when sampling. It measures the just-consumed- page's real bytes/row and re-issues `.list` from the page token with a size targeting `*sample-page-byte-budget*`+ page's real bytes/row and re-issues `.list` from the page token with a size targeting `*page-byte-budget*`-- a sliding window that adapts per page to the table's actual data instead of guessing from column types. Returnsnil once the `max-rows` budget is spent or there are no more pages."[^Table bq-table ^long max-rows]- (let [budget (long *sample-page-byte-budget*)+ (let [budget (long *page-byte-budget*)seen (atom {:bytes 0, :rows 0})](fn [^TableResult page](let [token (.getNextPageToken page)]@@ -612,7 +633,7 @@remaining (- max-rows (long rows))](when (pos? remaining)(*page-callback*)- (list-sample-page bq-table (next-sample-page-size budget bytes rows remaining) token))))))))+ (list-sample-page bq-table (next-page-size budget bytes rows remaining) token))))))))(defn- sample-table"Process a sample of rows of fields corresponding to the Metabase fields@@ -743,22 +764,36 @@;; realizing more rows as per the maximum result size(.setMaxResults *page-size*))))-(defn- next-page-same-size- "Default page-advance for [[reducible-bigquery-results]]: fetch the next page at the original request's page size- (BigQuery's `.getNextPage`). Returns nil once the result set is exhausted."- ^TableResult [^TableResult page]- (when (.hasNextPage page)- (log/trace "BigQuery: Fetching new page")- (*page-callback*)- (or (.getNextPage page)- (throw (ex-info "Cannot get next page from BigQuery" {})))))+(defn- adaptive-query-next-page+ "Adaptive page-advance for query-job results (the regular execution path), mirroring [[adaptive-sample-next-page]]+ but paging via `getQueryResults` -- the query result's own `.getNextPage` re-uses the original page size and can't+ be re-sized. Measures the just-consumed page's real bytes/row and re-issues the next page with a `pageSize`+ targeting [[*page-byte-budget*]], so a wide or heavy result fetches fewer rows per page instead of holding a+ large parsed page in memory. Returns nil once the result set is exhausted."+ [^BigQuery client job-id]+ (let [budget (long *page-byte-budget*)+ seen (atom {:bytes 0, :rows 0})]+ (fn [^TableResult page]+ (let [token (.getNextPageToken page)]+ (when-not (str/blank? token)+ (let [[page-bytes page-rows] (reduce (fn [[b n] row] [(+ (long b) (row-bytes row)) (inc (long n))])+ [0 0]+ (.getValues page))+ {:keys [bytes rows]} (swap! seen (fn [s] {:bytes (+ (long (:bytes s)) (long page-bytes))+ :rows (+ (long (:rows s)) (long page-rows))}))]+ (log/trace "BigQuery: Fetching new page")+ (*page-callback*)+ (.getQueryResults client job-id+ (u/varargs BigQuery$QueryResultsOption+ [(BigQuery$QueryResultsOption/pageSize+ (next-page-size budget bytes rows Long/MAX_VALUE))+ (BigQuery$QueryResultsOption/pageToken token)]))))))))(defn- reducible-bigquery-results- "Reducible over the rows of `page` and its successors. `next-page` is the swappable page-advance: given the- just-exhausted page it returns the next one (or nil when done). Defaults to [[next-page-same-size]] (fixed page- size); the sync sampler passes [[adaptive-sample-next-page]] to resize each page from measured bytes/row."- ([^TableResult page cancel-chan attempt-job-cancel-fn]- (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn next-page-same-size))+ "Reducible over the rows of `page` and its successors. `next-page` is the adaptive page-advance: given the+ just-exhausted page it measures it and returns the next one (re-sized from the measured bytes/row to a byte budget),+ or nil when done. Adaptive page sizing is the only mode -- callers pass [[adaptive-sample-next-page]] (sampling, via+ `.list`) or [[adaptive-query-next-page]] (query execution, via `getQueryResults`)."([^TableResult page cancel-chan attempt-job-cancel-fn next-page](reifyclojure.lang.IReduceInit@@ -825,7 +860,8 @@cols {:cols columns}results (eduction (map (fn [^FieldValueList row](mapv parse-field-value row parsers)))- (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn))]+ (reducible-bigquery-results page cancel-chan attempt-job-cancel-fn+ (adaptive-query-next-page client job-id)))](respond cols results)))(defn- execute-bigquerydiff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex a7f89c1dc6a..a20f316288b 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -159,8 +159,8 @@(is (= (+ (field-value-bytes (prim-cell "aa")) (field-value-bytes (prim-cell "bbb")))(row-bytes (field-value-list [(prim-cell "aa") (prim-cell "bbb")])))))))-(deftest ^:parallel next-sample-page-size-test- (let [next-size @#'bigquery/next-sample-page-size]+(deftest ^:parallel next-page-size-test+ (let [next-size @#'bigquery/next-page-size](testing "page size = budget / measured-avg-bytes-per-row";; 50 rows totalling 500 bytes -> avg 10 -> 1000/10 = 100(is (= 100 (next-size 1000 500 50 1000000))))@@ -181,7 +181,7 @@(deftest ^:parallel reducible-bigquery-results-nil-page-test(testing "a nil initial page reduces to an empty result instead of NPEing (#47339)"- (is (= [] (into [] (#'bigquery/reducible-bigquery-results nil nil (constantly nil)))))))+ (is (= [] (into [] (#'bigquery/reducible-bigquery-results nil nil (constantly nil) (constantly nil)))))))(deftest ^:synchronized adaptive-sample-next-page-test(let [requested (atom [])@@ -190,7 +190,7 @@(with-redefs [bigquery/list-sample-page (fn [_bq size _token](swap! requested conj size)(mock-page nil []))]- (binding [bigquery/*sample-page-byte-budget* budget]+ (binding [bigquery/*page-byte-budget* budget]((#'bigquery/adaptive-sample-next-page :table max-rows) (mock-page "tok" rows))(first @requested))))light (vec (repeatedly 5 #(field-value-list [(prim-cell "x")])))diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex f82f31dd9c6..1bda8877981 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -93,16 +93,13 @@(defmethod sql-jdbc.sync/describe-fields-pre-process-xf :redshift[_driver _db & _args]- (fn [rf]- (let [fields (volatile! (transient []))]- (fn- ([] (rf))- ([result]- (let [filtered (remove-duplicate-fields (persistent! @fields))]- (rf (reduce rf result filtered))))- ([result field]- (vswap! fields conj! field)- result)))))+ ;; `describe-fields-sql` orders by [table-schema table-name database-position], so each table's columns arrive+ ;; contiguously. A duplicate-column key is (table-schema, table-name, name) -- by definition all its occurrences are+ ;; within a single table -- so we can dedup per table with `partition-by` rather than buffering the entire result+ ;; set. This bounds memory to one table's columns (the per-table streaming contract) and is otherwise identical to a+ ;; global dedup.+ (comp (partition-by (juxt :table-schema :table-name))+ (mapcat remove-duplicate-fields)));; Skip the postgres implementation as it has to handle custom enums which redshift doesn't support.(defmethod driver/dynamic-database-types-lookup :redshift