The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BigQuery - ignore tables and fields with long names that we cannot insert into the app db (#76074)
Alexander Polyankinddbb6d05e5cb
diff --git a/src/metabase/sync/sync_metadata/fields/sync_instances.clj b/src/metabase/sync/sync_metadata/fields/sync_instances.cljindex 9985b583105..8396e86b251 100644--- a/src/metabase/sync/sync_metadata/fields/sync_instances.clj+++ b/src/metabase/sync/sync_metadata/fields/sync_instances.clj@@ -24,6 +24,33 @@;;; | CREATING / REACTIVATING FIELDS |;;; +----------------------------------------------------------------------------------------------------------------++(def ^:private field-name-max-length+ "Maximum length of the `metabase_field.name` column in the application DB (a `varchar(254)`). Fields whose name is+ longer than this can't be stored.++ We skip over-long Fields rather than store them, but an alternative worth considering is widening the column+ instead so these Fields become usable (e.g. BigQuery allows column names up to 300 characters). `name` is part of+ the `idx_unique_field` unique constraint and the `idx_field_name_lower` index, so any widening is bounded by+ MySQL/InnoDB's index key-length limit (~3072 bytes at utf8mb4 = 4 bytes/char) — i.e. up to roughly `varchar(512)`,+ which would cover every warehouse including BigQuery. Truncating is not an option: `name` is the real warehouse+ column name used to generate SQL, and truncating would break queries and could collide."+ 254)++(mu/defn- remove-fields-with-too-long-names :- [:set i/TableMetadataField]+ "Drop any Fields in `db-metadata` whose name is too long to store in the application DB (see+ `field-name-max-length`), logging a warning. A single over-long column name would otherwise fail the INSERT for the+ entire chunk of Fields it lands in and prevent the rest of the Table from syncing."+ [table :- i/TableInstance+ db-metadata :- [:set i/TableMetadataField]]+ (let [{too-long true, ok false} (group-by #(< field-name-max-length (count (:name %))) db-metadata)]+ (when (seq too-long)+ (log/warnf "Skipping %d Field(s) in %s whose name exceeds %d characters: %s"+ (count too-long)+ (sync-util/name-for-logging table)+ field-name-max-length+ (pr-str (sort (map :name too-long)))))+ (set ok)))+(mu/defn- matching-inactive-fields :- [:maybe [:sequential i/FieldInstance]]"Return inactive Metabase Fields that match any of the Fields described by `new-field-metadatas`, if any such Fieldsexist."@@ -224,14 +251,15 @@db-metadata :- [:set i/TableMetadataField]our-metadata :- [:set common/TableMetadataFieldWithID]parent-id :- common/ParentID]- ;; syncing the active instances makes important changes to `our-metadata` that need to be passed to recursive- ;; calls, such as adding new Fields or making inactive ones active again. Keep updated version returned by- ;; `sync-active-instances!`- (log/tracef "Syncing field instances for %s DB: %s, Existing: %s"- (sync-util/name-for-logging table)- (pr-str (sort (map common/canonical-name db-metadata)))- (pr-str (sort (map common/canonical-name our-metadata))))- (let [{:keys [num-updates our-metadata]} (sync-active-instances! table db-metadata our-metadata parent-id)]- (+ num-updates- (retire-fields! table db-metadata our-metadata)- (sync-nested-field-instances! table db-metadata our-metadata)))))+ (let [db-metadata (remove-fields-with-too-long-names table db-metadata)]+ ;; syncing the active instances makes important changes to `our-metadata` that need to be passed to recursive+ ;; calls, such as adding new Fields or making inactive ones active again. Keep updated version returned by+ ;; `sync-active-instances!`+ (log/tracef "Syncing field instances for %s DB: %s, Existing: %s"+ (sync-util/name-for-logging table)+ (pr-str (sort (map common/canonical-name db-metadata)))+ (pr-str (sort (map common/canonical-name our-metadata))))+ (let [{:keys [num-updates our-metadata]} (sync-active-instances! table db-metadata our-metadata parent-id)]+ (+ num-updates+ (retire-fields! table db-metadata our-metadata)+ (sync-nested-field-instances! table db-metadata our-metadata))))))diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.cljindex 2dda79f08ec..6798536efb6 100644--- a/src/metabase/sync/sync_metadata/tables.clj+++ b/src/metabase/sync/sync_metadata/tables.clj@@ -171,6 +171,43 @@;; TODO - should we make this logic case-insensitive like it is for fields?+(def ^:private table-name-max-length+ "Maximum length of the `metabase_table.name` column in the application DB (a `varchar(256)`). Tables whose name is+ longer than this can't be stored.++ As with Fields, an alternative would be to widen the column, but `name` is part of the `idx_unique_table` unique+ constraint whose key also includes the `varchar(254)` schema helper, so on MySQL/InnoDB there's only room to grow+ `name` to roughly `varchar(512)` — still short of e.g. BigQuery's 1024-character table names. Truncating is not an+ option either: `name` is the real warehouse table name used to generate SQL."+ 256)++(def ^:private table-schema-max-length+ "Maximum length of the `metabase_table.schema` column in the application DB (a `varchar(254)`). Tables in a+ schema/dataset whose name is longer than this can't be stored — BigQuery dataset names, for instance, can be up to+ 1024 characters."+ 254)++(defn- table-name-or-schema-too-long?+ "Whether `table`'s name or schema is too long to store in the application DB (see `table-name-max-length` /+ `table-schema-max-length`)."+ [{table-name :name, table-schema :schema}]+ (or (< table-name-max-length (count table-name))+ (< table-schema-max-length (count (or table-schema "")))))++(mu/defn- remove-tables-with-too-long-names :- [:set i/DatabaseMetadataTable]+ "Drop any tables in `db-metadata-tables` whose name or schema is too long to store in the application DB (see+ `table-name-max-length` / `table-schema-max-length`), logging a warning. A single over-long table name or schema+ would otherwise abort creation of all remaining new tables for the database."+ [database :- i/DatabaseInstance+ db-metadata-tables :- [:set i/DatabaseMetadataTable]]+ (let [{too-long true, ok false} (group-by table-name-or-schema-too-long? db-metadata-tables)]+ (when (seq too-long)+ (log/warnf "Skipping %d Table(s) in %s whose name or schema is too long to store: %s"+ (count too-long)+ (sync-util/name-for-logging database)+ (pr-str (sort (map (fn [{:keys [schema name]}] (str schema "." name)) too-long)))))+ (set ok)))+(mu/defn- create-or-reactivate-tables!"Create `new-tables` for database, or if they already exist, mark them as active."[database :- i/DatabaseInstance@@ -364,6 +401,9 @@;; determine what's changed between what info we have and what's in the DB(let [driver (driver.u/database->driver database)db-table-metadatas (table-set db-metadata)+ ;; drop tables whose names can't be stored in the app DB before they reach an INSERT and abort creation of+ ;; the remaining new tables for this database+ db-table-metadatas (remove-tables-with-too-long-names database db-table-metadatas);; DEV-1898: workspace-isolated tables (`ws_*`) live on the warehouse but must not;; become :model/Table rows. They back canonical Tables via remap, not their own identity.db-table-metadatas (ws.table-remapping/filter-workspace-side-tables db-table-metadatas (:id database))diff --git a/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj b/test/metabase/sync/sync_metadata/fields/sync_instances_test.cljindex 53c0bcf6f61..a8c92154e28 100644--- a/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj+++ b/test/metabase/sync/sync_metadata/fields/sync_instances_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase.sync.sync-metadata :as sync-metadata][metabase.sync.sync-metadata.fields :as sync-fields]+ [metabase.sync.sync-metadata.fields.sync-instances :as sync-instances][metabase.test :as mt][metabase.test.mock.toucanery :as toucanery][metabase.util :as u]@@ -188,3 +189,19 @@(deftest auto-cruft-fields-none-are-crufted-with-no-hit-pattern(run-cruft-test ["^it was the best$" "^of times it was$" "^the worst of times$"] {:normal 12}))++(deftest remove-fields-with-too-long-names-test+ (testing "Fields whose name is too long to store in the app DB are dropped, so they don't abort the whole batch"+ (let [remove-too-long @#'sync-instances/remove-fields-with-too-long-names+ table (t2/instance :model/Table {:id 1, :name "t", :schema "s"})+ field (fn [nm] {:name nm, :database-type "TEXT", :base-type :type/Text, :database-position 0})+ ok (field "short_name")+ too-long (field (apply str (repeat 300 "a")))]+ (testing "an over-long name is removed while the rest are kept"+ (is (= #{ok} (remove-too-long table #{ok too-long}))))+ (testing "everything is kept when all names fit"+ (is (= #{ok} (remove-too-long table #{ok}))))+ (testing "boundary: a name of exactly the max length is kept; one character longer is dropped"+ (let [at-limit (field (apply str (repeat 254 "a")))+ over-limit (field (apply str (repeat 255 "a")))]+ (is (= #{at-limit} (remove-too-long table #{at-limit over-limit}))))))))diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.cljindex 1442d9db9cb..2898f5a1a2f 100644--- a/test/metabase/sync/sync_metadata/tables_test.clj+++ b/test/metabase/sync/sync_metadata/tables_test.clj@@ -389,3 +389,26 @@(let [updated-table (t2/select-one :model/Table (:id existing-table))](is (= :ingested (:data_authority updated-table)))(is (:active updated-table)))))))))++(deftest remove-tables-with-too-long-names-test+ (testing "Tables whose name is too long to store in the app DB are dropped, so they don't abort the creation pass"+ (let [remove-too-long @#'sync-tables/remove-tables-with-too-long-names+ database (t2/instance :model/Database {:id 1, :name "db", :engine :h2})+ tbl (fn [nm] {:name nm, :schema "public"})+ ok (tbl "short_name")+ too-long (tbl (apply str (repeat 300 "a")))]+ (testing "an over-long name is removed while the rest are kept"+ (is (= #{ok} (remove-too-long database #{ok too-long}))))+ (testing "everything is kept when all names fit"+ (is (= #{ok} (remove-too-long database #{ok}))))+ (testing "boundary: a name of exactly the max length is kept; one character longer is dropped"+ (let [at-limit (tbl (apply str (repeat 256 "a")))+ over-limit (tbl (apply str (repeat 257 "a")))]+ (is (= #{at-limit} (remove-too-long database #{at-limit over-limit})))))+ (testing "a table whose schema (dataset) name is too long is also dropped"+ (let [long-schema {:name "t", :schema (apply str (repeat 300 "s"))}]+ (is (= #{ok} (remove-too-long database #{ok long-schema})))))+ (testing "boundary: a schema of exactly the max length is kept; one character longer is dropped"+ (let [at-limit {:name "t1", :schema (apply str (repeat 254 "s"))}+ over-limit {:name "t2", :schema (apply str (repeat 255 "s"))}]+ (is (= #{at-limit} (remove-too-long database #{at-limit over-limit}))))))))