Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BigQuery - ignore tables and fields with long names that we cannot insert into the app db (#76074)

Alexander Polyankinddbb6d05e5cb
diff --git a/src/metabase/sync/sync_metadata/fields/sync_instances.clj b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
index 9985b583105..8396e86b251 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_instances.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
@@ -24,6 +24,33 @@
;;; | CREATING / REACTIVATING FIELDS |
;;; +----------------------------------------------------------------------------------------------------------------+
+(def ^:private field-name-max-length
+ "Maximum length of the `metabase_field.name` column in the application DB (a `varchar(254)`). Fields whose name is
+ longer than this can't be stored.
+
+ We skip over-long Fields rather than store them, but an alternative worth considering is widening the column
+ instead so these Fields become usable (e.g. BigQuery allows column names up to 300 characters). `name` is part of
+ the `idx_unique_field` unique constraint and the `idx_field_name_lower` index, so any widening is bounded by
+ MySQL/InnoDB's index key-length limit (~3072 bytes at utf8mb4 = 4 bytes/char) — i.e. up to roughly `varchar(512)`,
+ which would cover every warehouse including BigQuery. Truncating is not an option: `name` is the real warehouse
+ column name used to generate SQL, and truncating would break queries and could collide."
+ 254)
+
+(mu/defn- remove-fields-with-too-long-names :- [:set i/TableMetadataField]
+ "Drop any Fields in `db-metadata` whose name is too long to store in the application DB (see
+ `field-name-max-length`), logging a warning. A single over-long column name would otherwise fail the INSERT for the
+ entire chunk of Fields it lands in and prevent the rest of the Table from syncing."
+ [table :- i/TableInstance
+ db-metadata :- [:set i/TableMetadataField]]
+ (let [{too-long true, ok false} (group-by #(< field-name-max-length (count (:name %))) db-metadata)]
+ (when (seq too-long)
+ (log/warnf "Skipping %d Field(s) in %s whose name exceeds %d characters: %s"
+ (count too-long)
+ (sync-util/name-for-logging table)
+ field-name-max-length
+ (pr-str (sort (map :name too-long)))))
+ (set ok)))
+
(mu/defn- matching-inactive-fields :- [:maybe [:sequential i/FieldInstance]]
"Return inactive Metabase Fields that match any of the Fields described by `new-field-metadatas`, if any such Fields
exist."
@@ -224,14 +251,15 @@
db-metadata :- [:set i/TableMetadataField]
our-metadata :- [:set common/TableMetadataFieldWithID]
parent-id :- common/ParentID]
- ;; syncing the active instances makes important changes to `our-metadata` that need to be passed to recursive
- ;; calls, such as adding new Fields or making inactive ones active again. Keep updated version returned by
- ;; `sync-active-instances!`
- (log/tracef "Syncing field instances for %s DB: %s, Existing: %s"
- (sync-util/name-for-logging table)
- (pr-str (sort (map common/canonical-name db-metadata)))
- (pr-str (sort (map common/canonical-name our-metadata))))
- (let [{:keys [num-updates our-metadata]} (sync-active-instances! table db-metadata our-metadata parent-id)]
- (+ num-updates
- (retire-fields! table db-metadata our-metadata)
- (sync-nested-field-instances! table db-metadata our-metadata)))))
+ (let [db-metadata (remove-fields-with-too-long-names table db-metadata)]
+ ;; syncing the active instances makes important changes to `our-metadata` that need to be passed to recursive
+ ;; calls, such as adding new Fields or making inactive ones active again. Keep updated version returned by
+ ;; `sync-active-instances!`
+ (log/tracef "Syncing field instances for %s DB: %s, Existing: %s"
+ (sync-util/name-for-logging table)
+ (pr-str (sort (map common/canonical-name db-metadata)))
+ (pr-str (sort (map common/canonical-name our-metadata))))
+ (let [{:keys [num-updates our-metadata]} (sync-active-instances! table db-metadata our-metadata parent-id)]
+ (+ num-updates
+ (retire-fields! table db-metadata our-metadata)
+ (sync-nested-field-instances! table db-metadata our-metadata))))))
diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.clj
index 2dda79f08ec..6798536efb6 100644
--- a/src/metabase/sync/sync_metadata/tables.clj
+++ b/src/metabase/sync/sync_metadata/tables.clj
@@ -171,6 +171,43 @@
;; TODO - should we make this logic case-insensitive like it is for fields?
+(def ^:private table-name-max-length
+ "Maximum length of the `metabase_table.name` column in the application DB (a `varchar(256)`). Tables whose name is
+ longer than this can't be stored.
+
+ As with Fields, an alternative would be to widen the column, but `name` is part of the `idx_unique_table` unique
+ constraint whose key also includes the `varchar(254)` schema helper, so on MySQL/InnoDB there's only room to grow
+ `name` to roughly `varchar(512)` — still short of e.g. BigQuery's 1024-character table names. Truncating is not an
+ option either: `name` is the real warehouse table name used to generate SQL."
+ 256)
+
+(def ^:private table-schema-max-length
+ "Maximum length of the `metabase_table.schema` column in the application DB (a `varchar(254)`). Tables in a
+ schema/dataset whose name is longer than this can't be stored — BigQuery dataset names, for instance, can be up to
+ 1024 characters."
+ 254)
+
+(defn- table-name-or-schema-too-long?
+ "Whether `table`'s name or schema is too long to store in the application DB (see `table-name-max-length` /
+ `table-schema-max-length`)."
+ [{table-name :name, table-schema :schema}]
+ (or (< table-name-max-length (count table-name))
+ (< table-schema-max-length (count (or table-schema "")))))
+
+(mu/defn- remove-tables-with-too-long-names :- [:set i/DatabaseMetadataTable]
+ "Drop any tables in `db-metadata-tables` whose name or schema is too long to store in the application DB (see
+ `table-name-max-length` / `table-schema-max-length`), logging a warning. A single over-long table name or schema
+ would otherwise abort creation of all remaining new tables for the database."
+ [database :- i/DatabaseInstance
+ db-metadata-tables :- [:set i/DatabaseMetadataTable]]
+ (let [{too-long true, ok false} (group-by table-name-or-schema-too-long? db-metadata-tables)]
+ (when (seq too-long)
+ (log/warnf "Skipping %d Table(s) in %s whose name or schema is too long to store: %s"
+ (count too-long)
+ (sync-util/name-for-logging database)
+ (pr-str (sort (map (fn [{:keys [schema name]}] (str schema "." name)) too-long)))))
+ (set ok)))
+
(mu/defn- create-or-reactivate-tables!
"Create `new-tables` for database, or if they already exist, mark them as active."
[database :- i/DatabaseInstance
@@ -364,6 +401,9 @@
;; determine what's changed between what info we have and what's in the DB
(let [driver (driver.u/database->driver database)
db-table-metadatas (table-set db-metadata)
+ ;; drop tables whose names can't be stored in the app DB before they reach an INSERT and abort creation of
+ ;; the remaining new tables for this database
+ db-table-metadatas (remove-tables-with-too-long-names database db-table-metadatas)
;; DEV-1898: workspace-isolated tables (`ws_*`) live on the warehouse but must not
;; become :model/Table rows. They back canonical Tables via remap, not their own identity.
db-table-metadatas (ws.table-remapping/filter-workspace-side-tables db-table-metadatas (:id database))
diff --git a/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj b/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj
index 53c0bcf6f61..a8c92154e28 100644
--- a/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase.sync.sync-metadata :as sync-metadata]
[metabase.sync.sync-metadata.fields :as sync-fields]
+ [metabase.sync.sync-metadata.fields.sync-instances :as sync-instances]
[metabase.test :as mt]
[metabase.test.mock.toucanery :as toucanery]
[metabase.util :as u]
@@ -188,3 +189,19 @@
(deftest auto-cruft-fields-none-are-crufted-with-no-hit-pattern
(run-cruft-test ["^it was the best$" "^of times it was$" "^the worst of times$"] {:normal 12}))
+
+(deftest remove-fields-with-too-long-names-test
+ (testing "Fields whose name is too long to store in the app DB are dropped, so they don't abort the whole batch"
+ (let [remove-too-long @#'sync-instances/remove-fields-with-too-long-names
+ table (t2/instance :model/Table {:id 1, :name "t", :schema "s"})
+ field (fn [nm] {:name nm, :database-type "TEXT", :base-type :type/Text, :database-position 0})
+ ok (field "short_name")
+ too-long (field (apply str (repeat 300 "a")))]
+ (testing "an over-long name is removed while the rest are kept"
+ (is (= #{ok} (remove-too-long table #{ok too-long}))))
+ (testing "everything is kept when all names fit"
+ (is (= #{ok} (remove-too-long table #{ok}))))
+ (testing "boundary: a name of exactly the max length is kept; one character longer is dropped"
+ (let [at-limit (field (apply str (repeat 254 "a")))
+ over-limit (field (apply str (repeat 255 "a")))]
+ (is (= #{at-limit} (remove-too-long table #{at-limit over-limit}))))))))
diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.clj
index 1442d9db9cb..2898f5a1a2f 100644
--- a/test/metabase/sync/sync_metadata/tables_test.clj
+++ b/test/metabase/sync/sync_metadata/tables_test.clj
@@ -389,3 +389,26 @@
(let [updated-table (t2/select-one :model/Table (:id existing-table))]
(is (= :ingested (:data_authority updated-table)))
(is (:active updated-table)))))))))
+
+(deftest remove-tables-with-too-long-names-test
+ (testing "Tables whose name is too long to store in the app DB are dropped, so they don't abort the creation pass"
+ (let [remove-too-long @#'sync-tables/remove-tables-with-too-long-names
+ database (t2/instance :model/Database {:id 1, :name "db", :engine :h2})
+ tbl (fn [nm] {:name nm, :schema "public"})
+ ok (tbl "short_name")
+ too-long (tbl (apply str (repeat 300 "a")))]
+ (testing "an over-long name is removed while the rest are kept"
+ (is (= #{ok} (remove-too-long database #{ok too-long}))))
+ (testing "everything is kept when all names fit"
+ (is (= #{ok} (remove-too-long database #{ok}))))
+ (testing "boundary: a name of exactly the max length is kept; one character longer is dropped"
+ (let [at-limit (tbl (apply str (repeat 256 "a")))
+ over-limit (tbl (apply str (repeat 257 "a")))]
+ (is (= #{at-limit} (remove-too-long database #{at-limit over-limit})))))
+ (testing "a table whose schema (dataset) name is too long is also dropped"
+ (let [long-schema {:name "t", :schema (apply str (repeat 300 "s"))}]
+ (is (= #{ok} (remove-too-long database #{ok long-schema})))))
+ (testing "boundary: a schema of exactly the max length is kept; one character longer is dropped"
+ (let [at-limit {:name "t1", :schema (apply str (repeat 254 "s"))}
+ over-limit {:name "t2", :schema (apply str (repeat 255 "s"))}]
+ (is (= #{at-limit} (remove-too-long database #{at-limit over-limit}))))))))