The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-4213: Remote Sync: importing a GUI model before the target's database schema sync completes stores untyped columns (#82945)
Eric Normand3eff0fac3d7f
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex b8dec30d0b5..157b97240c7 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.remote-sync.api-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.remote-sync.api-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all][diehard.core :as dh][java-time.api :as t]@@ -13,10 +14,12 @@[metabase-enterprise.remote-sync.source.git :as source.git][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase.driver.settings :as driver.settings][metabase.settings.core :as setting][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -206,6 +209,79 @@(is (=? {:status "success" :task_id int?} resp))(is (remote-sync.task/successful? completed-task))))))))+(defn- gui-model-yaml+ "YAML for a GUI model over `db-name`'s PUBLIC.VENUES, in the shape an export writes: the result metadata+ carries only the model overrides, with no `base_type` and no field `id`."+ [entity-id collection-id db-name]+ (-> (test-helpers/generate-card-yaml entity-id "Venues Model" collection-id "model")+ (str/replace "database_id: test-data (h2)" (str "database_id: " db-name))+ (str/replace "dataset_query: {}"+ (format (str "dataset_query:\n"+ " database: %s\n"+ " type: query\n"+ " query:\n"+ " source-table:\n"+ " - %s\n"+ " - PUBLIC\n"+ " - VENUES")+ db-name db-name))+ (str/replace "result_metadata: null"+ (str "result_metadata:\n"+ (str/join (for [[col-name display-name] [["ID" "ID"]+ ["NAME" "Venue Name"]+ ["CATEGORY_ID" "Category ID"]+ ["LATITUDE" "Latitude"]+ ["LONGITUDE" "Longitude"]+ ["PRICE" "Price"]]]+ (format "- display_name: %s\n name: %s\n visibility_type: normal\n"+ display-name col-name)))))))++(deftest import-gui-model-before-schema-sync-test+ (testing (str "GHY-4213: a GUI model imported before the target has synced its table stores untyped columns. "+ "Once the schema sync runs, the model's columns must get their types and field ids, so that "+ "the query builder offers typed filters on them.")+ (let [details (:details (mt/db))+ db-name (mt/random-name)+ collection-id "ghy4213collectionxxxx"+ card-eid "ghy4213modelxxxxxxxxx"]+ ;; Keep the new database from being synced when it is created, so its tables do not exist yet at import.+ (mt/with-temporary-setting-values [disable-auto-sync true+ remote-sync-url "https://github.com/test/repo.git"+ remote-sync-token "test-token"+ remote-sync-branch "main"]+ (test-helpers/commit-with-temp+ (fn []+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details details :name db-name}]+ (let [source (test-helpers/create-mock-source+ :initial-files {"main" {(format "collections/%s_ghy/%s_ghy.yaml" collection-id collection-id)+ (test-helpers/generate-collection-yaml collection-id "GHY 4213" :is-remote-synced true)+ (format "collections/%s_ghy/cards/%s_venues_model.yaml" collection-id card-eid)+ (gui-model-yaml card-eid collection-id db-name)}})]+ (try+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly source)]+ (let [{:keys [task_id]} (mt/user-http-request :crowberto :post 200 "ee/remote-sync/import"+ {:force true :expected_branch "main"})]+ (is (remote-sync.task/successful? (wait-for-task-completion task_id)))))+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (f)))]+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))+ (let [card-id (t2/select-one-pk :model/Card :entity_id card-eid)+ table-id (t2/select-one-pk :model/Table :db_id db-id :name "VENUES")+ field-id (fn [field-name]+ (t2/select-one-pk :model/Field :table_id table-id :name field-name))]+ (is (=? [{:name "ID" :base_type "type/BigInteger" :id (field-id "ID")}+ {:name "NAME" :base_type "type/Text" :id (field-id "NAME")+ :display_name "Venue Name"}+ {:name "CATEGORY_ID" :base_type "type/Integer" :id (field-id "CATEGORY_ID")}+ {:name "LATITUDE" :base_type "type/Float" :id (field-id "LATITUDE")}+ {:name "LONGITUDE" :base_type "type/Float" :id (field-id "LONGITUDE")}+ {:name "PRICE" :base_type "type/Integer" :id (field-id "PRICE")}]+ (:result_metadata (mt/user-http-request :crowberto :get 200 (str "card/" card-id))))))+ (finally+ (t2/delete! :model/Card :entity_id card-eid)+ (t2/delete! :model/Collection :entity_id collection-id)))))))))))+(deftest import-with-specific-branch-test(testing "POST /api/ee/remote-sync/import succeeds with specific branch"(let [mock-develop (test-helpers/create-mock-source :branch "develop")]diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.cljindex 9852fa51df2..a6a0cb09ab7 100644--- a/src/metabase/queries/db.clj+++ b/src/metabase/queries/db.clj@@ -99,6 +99,16 @@[card-ids :- [:sequential ::lib.schema.id/card]](t2/select [:model/Card :id :dataset_query :card_schema] :id [:in card-ids]))+(mu/defn mbql-model-cards+ "The IDs, types, queries, and result metadata of the unarchived MBQL models that query the Database with+ `database-id`."+ [database-id :- ::lib.schema.id/database]+ (t2/select [:model/Card :id :type :dataset_query :result_metadata :card_schema]+ :database_id database-id+ :type :model+ :query_type :query+ :archived false))+(mu/defn source-card-dependents"The IDs and source Card IDs of the Cards whose source Card is one of `source-card-ids`."[source-card-ids :- [:or [:set ::lib.schema.id/card] [:sequential ::lib.schema.id/card]]]diff --git a/src/metabase/queries/events/backfill_model_metadata_on_sync.clj b/src/metabase/queries/events/backfill_model_metadata_on_sync.cljnew file mode 100644index 00000000000..85d43df6c36--- /dev/null+++ b/src/metabase/queries/events/backfill_model_metadata_on_sync.clj@@ -0,0 +1,18 @@+(ns metabase.queries.events.backfill-model-metadata-on-sync+ (:require+ [metabase.events.core :as events]+ [metabase.queries.models.card.metadata :as card.metadata]+ [metabase.util.log :as log]+ [methodical.core :as methodical]))++(events/derive! ::event :metabase/event)+(events/derive! :event/sync-metadata-end ::event)++(methodical/defmethod events/publish-event! ::event+ "After a metadata sync of a Database, infer the metadata of its models that were stored untyped because their+ fields were not synced yet."+ [_topic {database-id :database_id}]+ (try+ (card.metadata/backfill-untyped-model-metadata! database-id)+ (catch Throwable e+ (log/errorf e "Error inferring untyped model metadata for Database %d" database-id))))diff --git a/src/metabase/queries/init.clj b/src/metabase/queries/init.cljindex d07c118a8c4..58003adc9a0 100644--- a/src/metabase/queries/init.clj+++ b/src/metabase/queries/init.clj@@ -1,5 +1,6 @@(ns metabase.queries.init(:require+ [metabase.queries.events.backfill-model-metadata-on-sync][metabase.queries.events.cards-notification-deleted-on-card-save][metabase.queries.events.schema];; for Malli-registered schemasdiff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.cljindex e3e39cf3407..3f4ecf26fb4 100644--- a/src/metabase/queries/models/card/metadata.clj+++ b/src/metabase/queries/models/card/metadata.clj@@ -214,6 +214,24 @@ saved later when it is ready.":metadata/own-model-query? true}))](infer-metadata query)))+(defn untyped-metadata?+ "Whether `metadata` has columns and none of them has a known base type."+ [metadata]+ (boolean (and (seq metadata)+ (every? #(contains? #{nil :type/*} (:base_type %)) metadata))))++(defn backfill-untyped-model-metadata!+ "Re-infer the result metadata of each MBQL model on the Database with `database-id` whose stored metadata is+ [[untyped-metadata?]], keeping the model overrides. A model whose metadata still cannot be inferred stays as it is."+ [database-id]+ ;; A model deserialized before its fields were synced stores only its overrides, because inference failed then.+ (doseq [card (queries.db/mbql-model-cards database-id)+ :when (untyped-metadata? (:result_metadata card))+ :let [metadata (infer-metadata-with-model-overrides (:dataset_query card) card)]+ :when (and metadata (not (untyped-metadata? metadata)))]+ (queries.db/update-card! (:id card) {:result_metadata metadata})+ (log/infof "Inferred the untyped result metadata of model Card %d" (:id card))))+;; TODO: Refactor this to use idents rather than names, so it's more robust.(defn refresh-metadata"Update cached result metadata to reflect changes to the underlying tables.diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.cljindex 2f1c553d8cd..4625a19d909 100644--- a/test/metabase/queries/models/card/metadata_test.clj+++ b/test/metabase/queries/models/card/metadata_test.clj@@ -5,7 +5,8 @@[metabase.lib.metadata :as lib.metadata][metabase.queries.models.card.metadata :as card.metadata][metabase.queries.schema :as queries.schema]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [toucan2.core :as t2]))(deftest ^:parallel populate-result-metadata-normalize-output-test(testing "populate-result-metadata should normalize output"@@ -61,3 +62,48 @@query (lib/query mp (lib.metadata/table mp (mt/id :orders)))](is (= 9 ;; there are two remaps, giving 11 if included(count (card.metadata/infer-metadata query))))))))++(deftest ^:parallel untyped-metadata?-test+ (testing "metadata is untyped when it has columns and none of them has a known base type (GHY-4213)"+ (are [expected metadata] (= expected (card.metadata/untyped-metadata? metadata))+ true [{:name "A"} {:name "B" :base_type :type/*}]+ false [{:name "A" :base_type :type/*} {:name "B" :base_type :type/Integer}]+ false [{:name "A" :base_type :type/Text}]+ false []+ false nil)))++(defn- venues-query []+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :venues)))))++(deftest backfill-untyped-model-metadata!-test+ (testing "backfill-untyped-model-metadata! infers the untyped metadata of the models on a database (GHY-4213)"+ (mt/with-temp [:model/Table {inactive-table-id :id} {:db_id (mt/id) :name "GHY_4213_NOT_SYNCED" :active false}+ :model/Card {untyped-id :id} {:type :model+ :dataset_query (venues-query)+ :result_metadata [{:name "NAME" :display_name "Venue Name" :base_type :type/*}]}+ :model/Card {typed-id :id} {:type :model+ :dataset_query (venues-query)+ :result_metadata [{:name "NAME" :display_name "Typed" :base_type :type/Text}]}+ :model/Card {unresolvable-id :id} {:type :model+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table inactive-table-id}}+ :result_metadata [{:name "NAME" :base_type :type/*}]}]+ (let [metadata (fn [card-id]+ (:result_metadata (t2/select-one [:model/Card :result_metadata :card_schema] :id card-id)))+ typed (metadata typed-id)+ unresolvable (metadata unresolvable-id)]+ (card.metadata/backfill-untyped-model-metadata! (mt/id))+ (testing "an untyped model gets its inferred types and field ids, keeping its overrides"+ (is (=? [{:name "ID"}+ {:name "NAME" :base_type :type/Text :id (mt/id :venues :name) :display_name "Venue Name"}+ {:name "CATEGORY_ID"}+ {:name "LATITUDE"}+ {:name "LONGITUDE"}+ {:name "PRICE" :base_type :type/Integer :id (mt/id :venues :price)}]+ (metadata untyped-id))))+ (testing "a model whose metadata is typed is not changed"+ (is (= typed (metadata typed-id))))+ (testing "a model whose metadata still cannot be inferred is not changed"+ (is (= unresolvable (metadata unresolvable-id))))))))