Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-4213: Remote Sync: importing a GUI model before the target's database schema sync completes stores untyped columns (#82945)

Eric Normand3eff0fac3d7f
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index b8dec30d0b5..157b97240c7 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.remote-sync.api-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.remote-sync.api-test]}}}}}}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[diehard.core :as dh]
[java-time.api :as t]
@@ -13,10 +14,12 @@
[metabase-enterprise.remote-sync.source.git :as source.git]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
+ [metabase.driver.settings :as driver.settings]
[metabase.settings.core :as setting]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
+ [metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -206,6 +209,79 @@
(is (=? {:status "success" :task_id int?} resp))
(is (remote-sync.task/successful? completed-task))))))))
+(defn- gui-model-yaml
+ "YAML for a GUI model over `db-name`'s PUBLIC.VENUES, in the shape an export writes: the result metadata
+ carries only the model overrides, with no `base_type` and no field `id`."
+ [entity-id collection-id db-name]
+ (-> (test-helpers/generate-card-yaml entity-id "Venues Model" collection-id "model")
+ (str/replace "database_id: test-data (h2)" (str "database_id: " db-name))
+ (str/replace "dataset_query: {}"
+ (format (str "dataset_query:\n"
+ " database: %s\n"
+ " type: query\n"
+ " query:\n"
+ " source-table:\n"
+ " - %s\n"
+ " - PUBLIC\n"
+ " - VENUES")
+ db-name db-name))
+ (str/replace "result_metadata: null"
+ (str "result_metadata:\n"
+ (str/join (for [[col-name display-name] [["ID" "ID"]
+ ["NAME" "Venue Name"]
+ ["CATEGORY_ID" "Category ID"]
+ ["LATITUDE" "Latitude"]
+ ["LONGITUDE" "Longitude"]
+ ["PRICE" "Price"]]]
+ (format "- display_name: %s\n name: %s\n visibility_type: normal\n"
+ display-name col-name)))))))
+
+(deftest import-gui-model-before-schema-sync-test
+ (testing (str "GHY-4213: a GUI model imported before the target has synced its table stores untyped columns. "
+ "Once the schema sync runs, the model's columns must get their types and field ids, so that "
+ "the query builder offers typed filters on them.")
+ (let [details (:details (mt/db))
+ db-name (mt/random-name)
+ collection-id "ghy4213collectionxxxx"
+ card-eid "ghy4213modelxxxxxxxxx"]
+ ;; Keep the new database from being synced when it is created, so its tables do not exist yet at import.
+ (mt/with-temporary-setting-values [disable-auto-sync true
+ remote-sync-url "https://github.com/test/repo.git"
+ remote-sync-token "test-token"
+ remote-sync-branch "main"]
+ (test-helpers/commit-with-temp
+ (fn []
+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details details :name db-name}]
+ (let [source (test-helpers/create-mock-source
+ :initial-files {"main" {(format "collections/%s_ghy/%s_ghy.yaml" collection-id collection-id)
+ (test-helpers/generate-collection-yaml collection-id "GHY 4213" :is-remote-synced true)
+ (format "collections/%s_ghy/cards/%s_venues_model.yaml" collection-id card-eid)
+ (gui-model-yaml card-eid collection-id db-name)}})]
+ (try
+ (mt/with-dynamic-fn-redefs [source/source-from-settings (constantly source)]
+ (let [{:keys [task_id]} (mt/user-http-request :crowberto :post 200 "ee/remote-sync/import"
+ {:force true :expected_branch "main"})]
+ (is (remote-sync.task/successful? (wait-for-task-completion task_id)))))
+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]
+ (binding [driver.settings/*allow-testing-h2-connections* true]
+ (f)))]
+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))
+ (let [card-id (t2/select-one-pk :model/Card :entity_id card-eid)
+ table-id (t2/select-one-pk :model/Table :db_id db-id :name "VENUES")
+ field-id (fn [field-name]
+ (t2/select-one-pk :model/Field :table_id table-id :name field-name))]
+ (is (=? [{:name "ID" :base_type "type/BigInteger" :id (field-id "ID")}
+ {:name "NAME" :base_type "type/Text" :id (field-id "NAME")
+ :display_name "Venue Name"}
+ {:name "CATEGORY_ID" :base_type "type/Integer" :id (field-id "CATEGORY_ID")}
+ {:name "LATITUDE" :base_type "type/Float" :id (field-id "LATITUDE")}
+ {:name "LONGITUDE" :base_type "type/Float" :id (field-id "LONGITUDE")}
+ {:name "PRICE" :base_type "type/Integer" :id (field-id "PRICE")}]
+ (:result_metadata (mt/user-http-request :crowberto :get 200 (str "card/" card-id))))))
+ (finally
+ (t2/delete! :model/Card :entity_id card-eid)
+ (t2/delete! :model/Collection :entity_id collection-id)))))))))))
+
(deftest import-with-specific-branch-test
(testing "POST /api/ee/remote-sync/import succeeds with specific branch"
(let [mock-develop (test-helpers/create-mock-source :branch "develop")]
diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.clj
index 9852fa51df2..a6a0cb09ab7 100644
--- a/src/metabase/queries/db.clj
+++ b/src/metabase/queries/db.clj
@@ -99,6 +99,16 @@
[card-ids :- [:sequential ::lib.schema.id/card]]
(t2/select [:model/Card :id :dataset_query :card_schema] :id [:in card-ids]))
+(mu/defn mbql-model-cards
+ "The IDs, types, queries, and result metadata of the unarchived MBQL models that query the Database with
+ `database-id`."
+ [database-id :- ::lib.schema.id/database]
+ (t2/select [:model/Card :id :type :dataset_query :result_metadata :card_schema]
+ :database_id database-id
+ :type :model
+ :query_type :query
+ :archived false))
+
(mu/defn source-card-dependents
"The IDs and source Card IDs of the Cards whose source Card is one of `source-card-ids`."
[source-card-ids :- [:or [:set ::lib.schema.id/card] [:sequential ::lib.schema.id/card]]]
diff --git a/src/metabase/queries/events/backfill_model_metadata_on_sync.clj b/src/metabase/queries/events/backfill_model_metadata_on_sync.clj
new file mode 100644
index 00000000000..85d43df6c36
--- /dev/null
+++ b/src/metabase/queries/events/backfill_model_metadata_on_sync.clj
@@ -0,0 +1,18 @@
+(ns metabase.queries.events.backfill-model-metadata-on-sync
+ (:require
+ [metabase.events.core :as events]
+ [metabase.queries.models.card.metadata :as card.metadata]
+ [metabase.util.log :as log]
+ [methodical.core :as methodical]))
+
+(events/derive! ::event :metabase/event)
+(events/derive! :event/sync-metadata-end ::event)
+
+(methodical/defmethod events/publish-event! ::event
+ "After a metadata sync of a Database, infer the metadata of its models that were stored untyped because their
+ fields were not synced yet."
+ [_topic {database-id :database_id}]
+ (try
+ (card.metadata/backfill-untyped-model-metadata! database-id)
+ (catch Throwable e
+ (log/errorf e "Error inferring untyped model metadata for Database %d" database-id))))
diff --git a/src/metabase/queries/init.clj b/src/metabase/queries/init.clj
index d07c118a8c4..58003adc9a0 100644
--- a/src/metabase/queries/init.clj
+++ b/src/metabase/queries/init.clj
@@ -1,5 +1,6 @@
(ns metabase.queries.init
(:require
+ [metabase.queries.events.backfill-model-metadata-on-sync]
[metabase.queries.events.cards-notification-deleted-on-card-save]
[metabase.queries.events.schema]
;; for Malli-registered schemas
diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.clj
index e3e39cf3407..3f4ecf26fb4 100644
--- a/src/metabase/queries/models/card/metadata.clj
+++ b/src/metabase/queries/models/card/metadata.clj
@@ -214,6 +214,24 @@ saved later when it is ready."
:metadata/own-model-query? true}))]
(infer-metadata query)))
+(defn untyped-metadata?
+ "Whether `metadata` has columns and none of them has a known base type."
+ [metadata]
+ (boolean (and (seq metadata)
+ (every? #(contains? #{nil :type/*} (:base_type %)) metadata))))
+
+(defn backfill-untyped-model-metadata!
+ "Re-infer the result metadata of each MBQL model on the Database with `database-id` whose stored metadata is
+ [[untyped-metadata?]], keeping the model overrides. A model whose metadata still cannot be inferred stays as it is."
+ [database-id]
+ ;; A model deserialized before its fields were synced stores only its overrides, because inference failed then.
+ (doseq [card (queries.db/mbql-model-cards database-id)
+ :when (untyped-metadata? (:result_metadata card))
+ :let [metadata (infer-metadata-with-model-overrides (:dataset_query card) card)]
+ :when (and metadata (not (untyped-metadata? metadata)))]
+ (queries.db/update-card! (:id card) {:result_metadata metadata})
+ (log/infof "Inferred the untyped result metadata of model Card %d" (:id card))))
+
;; TODO: Refactor this to use idents rather than names, so it's more robust.
(defn refresh-metadata
"Update cached result metadata to reflect changes to the underlying tables.
diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.clj
index 2f1c553d8cd..4625a19d909 100644
--- a/test/metabase/queries/models/card/metadata_test.clj
+++ b/test/metabase/queries/models/card/metadata_test.clj
@@ -5,7 +5,8 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.queries.models.card.metadata :as card.metadata]
[metabase.queries.schema :as queries.schema]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
(deftest ^:parallel populate-result-metadata-normalize-output-test
(testing "populate-result-metadata should normalize output"
@@ -61,3 +62,48 @@
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
(is (= 9 ;; there are two remaps, giving 11 if included
(count (card.metadata/infer-metadata query))))))))
+
+(deftest ^:parallel untyped-metadata?-test
+ (testing "metadata is untyped when it has columns and none of them has a known base type (GHY-4213)"
+ (are [expected metadata] (= expected (card.metadata/untyped-metadata? metadata))
+ true [{:name "A"} {:name "B" :base_type :type/*}]
+ false [{:name "A" :base_type :type/*} {:name "B" :base_type :type/Integer}]
+ false [{:name "A" :base_type :type/Text}]
+ false []
+ false nil)))
+
+(defn- venues-query []
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id :venues)))))
+
+(deftest backfill-untyped-model-metadata!-test
+ (testing "backfill-untyped-model-metadata! infers the untyped metadata of the models on a database (GHY-4213)"
+ (mt/with-temp [:model/Table {inactive-table-id :id} {:db_id (mt/id) :name "GHY_4213_NOT_SYNCED" :active false}
+ :model/Card {untyped-id :id} {:type :model
+ :dataset_query (venues-query)
+ :result_metadata [{:name "NAME" :display_name "Venue Name" :base_type :type/*}]}
+ :model/Card {typed-id :id} {:type :model
+ :dataset_query (venues-query)
+ :result_metadata [{:name "NAME" :display_name "Typed" :base_type :type/Text}]}
+ :model/Card {unresolvable-id :id} {:type :model
+ :dataset_query {:database (mt/id)
+ :type :query
+ :query {:source-table inactive-table-id}}
+ :result_metadata [{:name "NAME" :base_type :type/*}]}]
+ (let [metadata (fn [card-id]
+ (:result_metadata (t2/select-one [:model/Card :result_metadata :card_schema] :id card-id)))
+ typed (metadata typed-id)
+ unresolvable (metadata unresolvable-id)]
+ (card.metadata/backfill-untyped-model-metadata! (mt/id))
+ (testing "an untyped model gets its inferred types and field ids, keeping its overrides"
+ (is (=? [{:name "ID"}
+ {:name "NAME" :base_type :type/Text :id (mt/id :venues :name) :display_name "Venue Name"}
+ {:name "CATEGORY_ID"}
+ {:name "LATITUDE"}
+ {:name "LONGITUDE"}
+ {:name "PRICE" :base_type :type/Integer :id (mt/id :venues :price)}]
+ (metadata untyped-id))))
+ (testing "a model whose metadata is typed is not changed"
+ (is (= typed (metadata typed-id))))
+ (testing "a model whose metadata still cannot be inferred is not changed"
+ (is (= unresolvable (metadata unresolvable-id))))))))