The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix source replacement with null effective_type (#72124)
Alexander Polyankin25969cb31ea4
diff --git a/frontend/src/metabase-lib/query/column_types.ts b/frontend/src/metabase-lib/query/column_types.tsindex b1b0b5afb4b..1cf1f61bd51 100644--- a/frontend/src/metabase-lib/query/column_types.ts+++ b/frontend/src/metabase-lib/query/column_types.ts@@ -55,5 +55,5 @@ export function isAssignableType(column1: ColumnMetadata | ColumnTypeInfo,column2: ColumnMetadata | ColumnTypeInfo,): boolean {- return ML.valid_filter_for_QMARK_(column1, column2);+ return ML.compatible_type_QMARK_(column1, column2);}diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljsindex ebeb02e8715..951520de838 100644--- a/src/metabase/lib/js.cljs+++ b/src/metabase/lib/js.cljs@@ -2611,12 +2611,12 @@(lib.core/with-wrapped-native-query a-query stage-number card-idlib.core/update-temporal-filter temporal-column start end))-(defn ^:export valid-filter-for?- "Given two columns, returns true if `src-column` is a valid source to use for filtering `dst-column`.+(defn ^:export compatible-type?+ "Given two columns, returns true if they have compatible types.> **Code health:** Healthy."[src-column dst-column]- (lib.types.isa/valid-filter-for? src-column dst-column))+ (lib.types.isa/compatible-type? src-column dst-column))(defn ^:export dependent-metadata"Return a JS array of entities which `a-query` requires to be loaded. `card-id` is provideddiff --git a/src/metabase/lib/types/isa.cljc b/src/metabase/lib/types/isa.cljcindex 675f6e8550c..10f594f1e5c 100644--- a/src/metabase/lib/types/isa.cljc+++ b/src/metabase/lib/types/isa.cljc@@ -154,6 +154,11 @@[column](clojure.core/isa? (column-type column) :type/Date))+(defn ^:export date-with-time?+ "Is `column` a datetime (date with time)?"+ [column]+ (clojure.core/isa? (column-type column) :type/DateTime))+(defn ^:export creation-timestamp?"Is `column` a creation timestamp column?"[column]@@ -267,14 +272,17 @@(or (clojure.core/isa? col-type :type/Text)(clojure.core/isa? col-type :type/TextLike))))-(defn valid-filter-for?- "Given two CLJS `:metadata/columns` returns true if `src-column` is a valid source to use for filtering `dst-column`.+(defn compatible-type?+ "Given two columns, returns true if they have compatible types.- That's the case if both are from the same family (strings, numbers, temporal) or if the `src-column` [[isa?]] subtype- of `dst-column`."+ That's the case if both are from the same family (strings, numbers, dates/datetimes, times, booleans) or if+ `src-column`'s base-type is a subtype of `dst-column`'s base-type."[src-column dst-column](or(and (string? src-column) (string? dst-column))(and (numeric? src-column) (numeric? dst-column))- (and (temporal? src-column) (temporal? dst-column))+ (and (date-without-time? src-column) (date-or-datetime? dst-column))+ (and (date-with-time? src-column) (date-with-time? dst-column))+ (and (time? src-column) (time? dst-column))+ (and (boolean? src-column) (boolean? dst-column))(clojure.core/isa? (:base-type src-column) (:base-type dst-column))))diff --git a/src/metabase/source_swap/compatibility.clj b/src/metabase/source_swap/compatibility.cljindex f37d57688c5..f027903aa3c 100644--- a/src/metabase/source_swap/compatibility.clj+++ b/src/metabase/source_swap/compatibility.clj@@ -18,7 +18,7 @@old-source-type :- ::source-swap.schema/source-typenew-source-type :- ::source-swap.schema/source-type](cond-> []- (not= (:effective-type old-column) (:effective-type new-column))+ (not (lib.types.isa/compatible-type? old-column new-column))(conj :column-type-mismatch)(and (= old-source-type :table)diff --git a/test/metabase/lib/types/isa_test.cljc b/test/metabase/lib/types/isa_test.cljcindex 95cf20211dd..dcaa2c44c04 100644--- a/test/metabase/lib/types/isa_test.cljc+++ b/test/metabase/lib/types/isa_test.cljc@@ -146,9 +146,9 @@(is (true? (lib.types.isa/numeric? {:effective-type :type/Float :semantic-type :type/Price})))(is (false? (lib.types.isa/numeric? {:effective-type :type/Text :semantic-type :type/Price}))))-(deftest ^:parallel valid-filter-for?-test+(deftest ^:parallel compatible-type?-test#_{:clj-kondo/ignore [:equals-true]}- (are [exp base-lhs eff-lhs base-rhs eff-rhs] (= exp (lib.types.isa/valid-filter-for?+ (are [exp base-lhs eff-lhs base-rhs eff-rhs] (= exp (lib.types.isa/compatible-type?{:base-type base-lhs:effective-type eff-lhs}{:base-type base-rhs@@ -162,12 +162,24 @@true :type/Integer :type/Quantity :type/Float :type/Numbertrue :type/Float :type/Number :type/Float :type/Price- true :type/DateTime :type/Temporal :type/Time :type/Temporal+ true :type/Date :type/Date :type/Date :type/Date+ true :type/Date :type/Date :type/DateTime :type/DateTime+ false :type/DateTime :type/DateTime :type/Date :type/Date+ true :type/DateTime :type/DateTime :type/DateTime :type/DateTimeWithTZ+ true :type/Time :type/Time :type/Time :type/TimeWithTZ++ true :type/Boolean :type/Boolean :type/Boolean :type/Booleanfalse :type/String :type/Text :type/Integer :type/Numberfalse :type/Integer :type/Number :type/String :type/Text- false :type/DateTime :type/Temporal :type/String :type/Text- false :type/String :type/Text :type/DateTime :type/Temporal))+ false :type/DateTime :type/DateTime :type/String :type/Text+ false :type/String :type/Text :type/DateTime :type/DateTime+ false :type/Boolean :type/Boolean :type/String :type/Text+ false :type/DateTime :type/DateTime :type/Time :type/Time+ false :type/Time :type/Time :type/Date :type/Date++ false :type/PostgresEnum :type/PostgresEnum :type/PostgresBitString :type/PostgresBitString+ false :type/PostgresBitString :type/PostgresBitString :type/PostgresEnum :type/PostgresEnum))(deftest ^:parallel effective-type-fallback-test(are [expected predicate column] (= expected (predicate column))@@ -188,6 +200,12 @@false lib.types.isa/date-without-time? {:effective-type :type/String :base-type :type/String}false lib.types.isa/date-without-time? {:effective-type :type/DateTime :base-type :type/String}+ true lib.types.isa/date-with-time? {:base-type :type/DateTime}+ true lib.types.isa/date-with-time? {:effective-type :type/DateTime :base-type :type/String}+ false lib.types.isa/date-with-time? {:base-type :type/Date}+ false lib.types.isa/date-with-time? {:base-type :type/String}+ false lib.types.isa/date-with-time? {:effective-type :type/Date :base-type :type/String}+true lib.types.isa/time? {:base-type :type/Time}true lib.types.isa/time? {:effective-type :type/Time :base-type :type/String}diff --git a/test/metabase/source_swap/compatibility_test.clj b/test/metabase/source_swap/compatibility_test.cljindex cb31c3f42e0..230d082fd85 100644--- a/test/metabase/source_swap/compatibility_test.clj+++ b/test/metabase/source_swap/compatibility_test.clj@@ -7,51 +7,102 @@(set! *warn-on-reflection* true)-(def ^:private mp- (lib.tu/mock-metadata-provider- {:database {:id 1 :name "db" :engine :h2}- :tables [{:id 100 :name "t" :db-id 1}]- :fields [{:id 1 :table-id 100 :name "PLAIN"- :base-type :type/Integer :effective-type :type/Integer}- {:id 2 :table-id 100 :name "DIFFERENT_TYPE"- :base-type :type/Text :effective-type :type/Text}- {:id 3 :table-id 100 :name "PK"- :base-type :type/Integer :effective-type :type/Integer- :semantic-type :type/PK}- {:id 4 :table-id 100 :name "FK_A"- :base-type :type/Integer :effective-type :type/Integer- :semantic-type :type/FK :fk-target-field-id 999}- {:id 5 :table-id 100 :name "FK_B"- :base-type :type/Integer :effective-type :type/Integer- :semantic-type :type/FK :fk-target-field-id 888}- {:id 6 :table-id 100 :name "NOT_FK"- :base-type :type/Integer :effective-type :type/Integer}]}))--(defn- field [id]+(defn- field [mp id](lib.metadata/field mp id))(deftest ^:parallel column-errors-no-errors-test- (testing "should return empty errors for matching columns"- (is (= []- (source-swap.compatibility/column-errors (field 1) (field 1) :table :table)))))+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "quantity"+ :base-type :type/Integer :effective-type :type/Integer}+ {:id 2 :table-id 200 :name "quantity"+ :base-type :type/Integer :effective-type :type/Integer}]})]+ (testing "should return empty errors for matching columns"+ (is (= []+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))(deftest ^:parallel column-errors-type-mismatch-test- (testing "should detect column type mismatch"- (is (= [:column-type-mismatch]- (source-swap.compatibility/column-errors (field 1) (field 2) :table :table)))))+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "quantity"+ :base-type :type/Integer :effective-type :type/Integer}+ {:id 2 :table-id 200 :name "quantity"+ :base-type :type/Text :effective-type :type/Text}]})]+ (testing "should detect column type mismatch"+ (is (= [:column-type-mismatch]+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))(deftest ^:parallel column-errors-missing-primary-key-test- (testing "should detect missing primary key for table->table"- (is (= [:column-type-mismatch :missing-primary-key]- (source-swap.compatibility/column-errors (field 3) (field 2) :table :table))))- (testing "should not flag missing primary key for table->card"- (is (= [:column-type-mismatch]- (source-swap.compatibility/column-errors (field 3) (field 2) :table :card)))))+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "id"+ :base-type :type/Integer :effective-type :type/Integer+ :semantic-type :type/PK}+ {:id 2 :table-id 200 :name "id"+ :base-type :type/Text :effective-type :type/Text}]})]+ (testing "should detect missing primary key for table->table"+ (is (= [:column-type-mismatch :missing-primary-key]+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))+ (testing "should not flag missing primary key for table->card"+ (is (= [:column-type-mismatch]+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :card))))))(deftest ^:parallel column-errors-foreign-key-test- (testing "should detect missing foreign key"- (is (= [:missing-foreign-key]- (source-swap.compatibility/column-errors (field 4) (field 6) :table :table))))- (testing "should detect foreign key target mismatch"- (is (= [:foreign-key-mismatch]- (source-swap.compatibility/column-errors (field 4) (field 5) :table :table)))))+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}+ {:id 300 :name "invoices" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "user_id"+ :base-type :type/Integer :effective-type :type/Integer+ :semantic-type :type/FK :fk-target-field-id 999}+ {:id 2 :table-id 200 :name "user_id"+ :base-type :type/Integer :effective-type :type/Integer+ :semantic-type :type/FK :fk-target-field-id 888}+ {:id 3 :table-id 300 :name "user_id"+ :base-type :type/Integer :effective-type :type/Integer}]})]+ (testing "should detect missing foreign key"+ (is (= [:missing-foreign-key]+ (source-swap.compatibility/column-errors (field mp 1) (field mp 3) :table :table))))+ (testing "should detect foreign key target mismatch"+ (is (= [:foreign-key-mismatch]+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))++(deftest ^:parallel column-errors-nil-effective-type-test+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}+ {:id 300 :name "invoices" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "quantity"+ :base-type :type/Integer :effective-type :type/Integer}+ {:id 2 :table-id 200 :name "quantity"+ :base-type :type/Integer}+ {:id 3 :table-id 300 :name "quantity"+ :base-type :type/Text}]})]+ (testing "same base_type, nil effective_type should be compatible"+ (is (= []+ (source-swap.compatibility/column-errors (field mp 2) (field mp 1) :table :table))))+ (testing "different base_type, nil effective_type should be incompatible"+ (is (= [:column-type-mismatch]+ (source-swap.compatibility/column-errors (field mp 2) (field mp 3) :table :table))))))++(deftest ^:parallel column-errors-coerced-effective-type-test+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :engine :h2}+ :tables [{:id 100 :name "orders" :db-id 1}+ {:id 200 :name "transactions" :db-id 1}]+ :fields [{:id 1 :table-id 100 :name "created_at"+ :base-type :type/DateTime :effective-type :type/DateTime}+ {:id 2 :table-id 200 :name "created_at"+ :base-type :type/Text :effective-type :type/DateTime+ :coercion-strategy :Coercion/ISO8601->DateTime}]})]+ (testing "text column coerced to DateTime should be compatible with native DateTime"+ (is (= []+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))