The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Truncate data_type in BigQuery describe-fields SQL to prevent sync OOM (#78069)
dpsutton64f32ddea7f6
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 1c1d3b93d47..d911962e1af 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -458,6 +458,15 @@(sort-by (juxt :table-name :database-position :name)(describe-dataset-rows nested-lookup dataset-id table-name columns))))+(def ^:private max-data-type-length+ "Truncate `INFORMATION_SCHEMA` `data_type` strings to this many characters, in SQL. Only short prefixes are ever+ consumed (see [[raw-type->database+base-type]]; the longest verbatim type is `RANGE<TIMESTAMP>`), but a STRUCT+ column's `data_type` spells out its entire nested schema, and the `COLUMN_FIELD_PATHS` join repeats it on every+ nested-leaf row -- O(n^2) bytes for a column with n leaves. Dynamic-key STRUCTs (e.g. log-sink tables whose JSON+ payloads use user IDs as keys) can reach thousands of leaves with ~270KB type strings, which OOMed sync before this+ truncation."+ 200)+(defn- describe-dataset-fields-reducible"Reducibly describe the fields (including nested STRUCT fields) of `table-names` within `dataset-id`.@@ -476,10 +485,12 @@(query-honeysql driver database{:select [[:c.table_name :table_name][:c.column_name :column_name]- [:c.data_type :data_type]+ ;; truncated in SQL: a STRUCT's data_type repeats its whole nested schema+ ;; on every leaf row of the join, O(n^2) bytes -- see [[max-data-type-length]]+ [[:substr :c.data_type 1 max-data-type-length] :data_type][:c.ordinal_position :ordinal_position][[:= :c.is_partitioning_column "YES"] :partitioned]- [:p.data_type :nested_data_type]+ [[:substr :p.data_type 1 max-data-type-length] :nested_data_type][:p.field_path :field_path]]:from [[(information-schema-table project-id dataset-id "COLUMNS") :c]]:left-join [[(information-schema-table project-id dataset-id "COLUMN_FIELD_PATHS") :p]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex f61f8b58501..d38a29c0c7a 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -335,6 +335,24 @@(is (<= 22000 (count (into [] (driver/describe-fields :bigquery-cloud-sdk (mt/db))))))(is (<= 20 @invocation-count))))))))))+(deftest ^:parallel describe-fields-truncates-data-type-test+ (mt/test-driver :bigquery-cloud-sdk+ (testing "describe-fields truncates data_type in SQL: a STRUCT's data_type spells out its whole nested schema and+ the COLUMN_FIELD_PATHS join repeats it on every nested-leaf row, which OOMs sync on dynamic-key schemas"+ (let [captured-sql (atom nil)+ captured-params (atom nil)]+ (binding [bigquery/*process-native* (fn [respond _database sql params _cancel-chan]+ (reset! captured-sql sql)+ (reset! captured-params params)+ (respond {:cols []} []))]+ (is (= [] (into [] (#'bigquery/describe-dataset-fields-reducible+ :bigquery-cloud-sdk nil "some-project" "some_dataset" ["some_table"]))))+ (let [sql (-> @captured-sql u/lower-case-en (str/replace "`" ""))]+ (is (str/includes? sql "substr(c.data_type, ?, ?)"))+ (is (str/includes? sql "substr(p.data_type, ?, ?)"))+ ;; the "YES" between the pairs is the is_partitioning_column comparison+ (is (= [1 200 "YES" 1 200] (take 5 @captured-params)))))))))+(def ^:private native-dataset(tx/native-dataset-definition"native-dataset"