Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Truncate data_type in BigQuery describe-fields SQL to prevent sync OOM (#78069)

dpsutton64f32ddea7f6
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 1c1d3b93d47..d911962e1af 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -458,6 +458,15 @@
(sort-by (juxt :table-name :database-position :name)
(describe-dataset-rows nested-lookup dataset-id table-name columns))))
+(def ^:private max-data-type-length
+ "Truncate `INFORMATION_SCHEMA` `data_type` strings to this many characters, in SQL. Only short prefixes are ever
+ consumed (see [[raw-type->database+base-type]]; the longest verbatim type is `RANGE<TIMESTAMP>`), but a STRUCT
+ column's `data_type` spells out its entire nested schema, and the `COLUMN_FIELD_PATHS` join repeats it on every
+ nested-leaf row -- O(n^2) bytes for a column with n leaves. Dynamic-key STRUCTs (e.g. log-sink tables whose JSON
+ payloads use user IDs as keys) can reach thousands of leaves with ~270KB type strings, which OOMed sync before this
+ truncation."
+ 200)
+
(defn- describe-dataset-fields-reducible
"Reducibly describe the fields (including nested STRUCT fields) of `table-names` within `dataset-id`.
@@ -476,10 +485,12 @@
(query-honeysql driver database
{:select [[:c.table_name :table_name]
[:c.column_name :column_name]
- [:c.data_type :data_type]
+ ;; truncated in SQL: a STRUCT's data_type repeats its whole nested schema
+ ;; on every leaf row of the join, O(n^2) bytes -- see [[max-data-type-length]]
+ [[:substr :c.data_type 1 max-data-type-length] :data_type]
[:c.ordinal_position :ordinal_position]
[[:= :c.is_partitioning_column "YES"] :partitioned]
- [:p.data_type :nested_data_type]
+ [[:substr :p.data_type 1 max-data-type-length] :nested_data_type]
[:p.field_path :field_path]]
:from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]
:left-join [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :p]
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index f61f8b58501..d38a29c0c7a 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -335,6 +335,24 @@
(is (<= 22000 (count (into [] (driver/describe-fields :bigquery-cloud-sdk (mt/db))))))
(is (<= 20 @invocation-count))))))))))
+(deftest ^:parallel describe-fields-truncates-data-type-test
+ (mt/test-driver :bigquery-cloud-sdk
+ (testing "describe-fields truncates data_type in SQL: a STRUCT's data_type spells out its whole nested schema and
+ the COLUMN_FIELD_PATHS join repeats it on every nested-leaf row, which OOMs sync on dynamic-key schemas"
+ (let [captured-sql (atom nil)
+ captured-params (atom nil)]
+ (binding [bigquery/*process-native* (fn [respond _database sql params _cancel-chan]
+ (reset! captured-sql sql)
+ (reset! captured-params params)
+ (respond {:cols []} []))]
+ (is (= [] (into [] (#'bigquery/describe-dataset-fields-reducible
+ :bigquery-cloud-sdk nil "some-project" "some_dataset" ["some_table"]))))
+ (let [sql (-> @captured-sql u/lower-case-en (str/replace "`" ""))]
+ (is (str/includes? sql "substr(c.data_type, ?, ?)"))
+ (is (str/includes? sql "substr(p.data_type, ?, ?)"))
+ ;; the "YES" between the pairs is the is_partitioning_column comparison
+ (is (= [1 200 "YES" 1 200] (take 5 @captured-params)))))))))
+
(def ^:private native-dataset
(tx/native-dataset-definition
"native-dataset"