Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix source replacement with null effective_type (#72124)

Alexander Polyankin25969cb31ea4
diff --git a/frontend/src/metabase-lib/query/column_types.ts b/frontend/src/metabase-lib/query/column_types.ts
index b1b0b5afb4b..1cf1f61bd51 100644
--- a/frontend/src/metabase-lib/query/column_types.ts
+++ b/frontend/src/metabase-lib/query/column_types.ts
@@ -55,5 +55,5 @@ export function isAssignableType(
column1: ColumnMetadata | ColumnTypeInfo,
column2: ColumnMetadata | ColumnTypeInfo,
): boolean {
- return ML.valid_filter_for_QMARK_(column1, column2);
+ return ML.compatible_type_QMARK_(column1, column2);
}
diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljs
index ebeb02e8715..951520de838 100644
--- a/src/metabase/lib/js.cljs
+++ b/src/metabase/lib/js.cljs
@@ -2611,12 +2611,12 @@
(lib.core/with-wrapped-native-query a-query stage-number card-id
lib.core/update-temporal-filter temporal-column start end))
-(defn ^:export valid-filter-for?
- "Given two columns, returns true if `src-column` is a valid source to use for filtering `dst-column`.
+(defn ^:export compatible-type?
+ "Given two columns, returns true if they have compatible types.
> **Code health:** Healthy."
[src-column dst-column]
- (lib.types.isa/valid-filter-for? src-column dst-column))
+ (lib.types.isa/compatible-type? src-column dst-column))
(defn ^:export dependent-metadata
"Return a JS array of entities which `a-query` requires to be loaded. `card-id` is provided
diff --git a/src/metabase/lib/types/isa.cljc b/src/metabase/lib/types/isa.cljc
index 675f6e8550c..10f594f1e5c 100644
--- a/src/metabase/lib/types/isa.cljc
+++ b/src/metabase/lib/types/isa.cljc
@@ -154,6 +154,11 @@
[column]
(clojure.core/isa? (column-type column) :type/Date))
+(defn ^:export date-with-time?
+ "Is `column` a datetime (date with time)?"
+ [column]
+ (clojure.core/isa? (column-type column) :type/DateTime))
+
(defn ^:export creation-timestamp?
"Is `column` a creation timestamp column?"
[column]
@@ -267,14 +272,17 @@
(or (clojure.core/isa? col-type :type/Text)
(clojure.core/isa? col-type :type/TextLike))))
-(defn valid-filter-for?
- "Given two CLJS `:metadata/columns` returns true if `src-column` is a valid source to use for filtering `dst-column`.
+(defn compatible-type?
+ "Given two columns, returns true if they have compatible types.
- That's the case if both are from the same family (strings, numbers, temporal) or if the `src-column` [[isa?]] subtype
- of `dst-column`."
+ That's the case if both are from the same family (strings, numbers, dates/datetimes, times, booleans) or if
+ `src-column`'s base-type is a subtype of `dst-column`'s base-type."
[src-column dst-column]
(or
(and (string? src-column) (string? dst-column))
(and (numeric? src-column) (numeric? dst-column))
- (and (temporal? src-column) (temporal? dst-column))
+ (and (date-without-time? src-column) (date-or-datetime? dst-column))
+ (and (date-with-time? src-column) (date-with-time? dst-column))
+ (and (time? src-column) (time? dst-column))
+ (and (boolean? src-column) (boolean? dst-column))
(clojure.core/isa? (:base-type src-column) (:base-type dst-column))))
diff --git a/src/metabase/source_swap/compatibility.clj b/src/metabase/source_swap/compatibility.clj
index f37d57688c5..f027903aa3c 100644
--- a/src/metabase/source_swap/compatibility.clj
+++ b/src/metabase/source_swap/compatibility.clj
@@ -18,7 +18,7 @@
old-source-type :- ::source-swap.schema/source-type
new-source-type :- ::source-swap.schema/source-type]
(cond-> []
- (not= (:effective-type old-column) (:effective-type new-column))
+ (not (lib.types.isa/compatible-type? old-column new-column))
(conj :column-type-mismatch)
(and (= old-source-type :table)
diff --git a/test/metabase/lib/types/isa_test.cljc b/test/metabase/lib/types/isa_test.cljc
index 95cf20211dd..dcaa2c44c04 100644
--- a/test/metabase/lib/types/isa_test.cljc
+++ b/test/metabase/lib/types/isa_test.cljc
@@ -146,9 +146,9 @@
(is (true? (lib.types.isa/numeric? {:effective-type :type/Float :semantic-type :type/Price})))
(is (false? (lib.types.isa/numeric? {:effective-type :type/Text :semantic-type :type/Price}))))
-(deftest ^:parallel valid-filter-for?-test
+(deftest ^:parallel compatible-type?-test
#_{:clj-kondo/ignore [:equals-true]}
- (are [exp base-lhs eff-lhs base-rhs eff-rhs] (= exp (lib.types.isa/valid-filter-for?
+ (are [exp base-lhs eff-lhs base-rhs eff-rhs] (= exp (lib.types.isa/compatible-type?
{:base-type base-lhs
:effective-type eff-lhs}
{:base-type base-rhs
@@ -162,12 +162,24 @@
true :type/Integer :type/Quantity :type/Float :type/Number
true :type/Float :type/Number :type/Float :type/Price
- true :type/DateTime :type/Temporal :type/Time :type/Temporal
+ true :type/Date :type/Date :type/Date :type/Date
+ true :type/Date :type/Date :type/DateTime :type/DateTime
+ false :type/DateTime :type/DateTime :type/Date :type/Date
+ true :type/DateTime :type/DateTime :type/DateTime :type/DateTimeWithTZ
+ true :type/Time :type/Time :type/Time :type/TimeWithTZ
+
+ true :type/Boolean :type/Boolean :type/Boolean :type/Boolean
false :type/String :type/Text :type/Integer :type/Number
false :type/Integer :type/Number :type/String :type/Text
- false :type/DateTime :type/Temporal :type/String :type/Text
- false :type/String :type/Text :type/DateTime :type/Temporal))
+ false :type/DateTime :type/DateTime :type/String :type/Text
+ false :type/String :type/Text :type/DateTime :type/DateTime
+ false :type/Boolean :type/Boolean :type/String :type/Text
+ false :type/DateTime :type/DateTime :type/Time :type/Time
+ false :type/Time :type/Time :type/Date :type/Date
+
+ false :type/PostgresEnum :type/PostgresEnum :type/PostgresBitString :type/PostgresBitString
+ false :type/PostgresBitString :type/PostgresBitString :type/PostgresEnum :type/PostgresEnum))
(deftest ^:parallel effective-type-fallback-test
(are [expected predicate column] (= expected (predicate column))
@@ -188,6 +200,12 @@
false lib.types.isa/date-without-time? {:effective-type :type/String :base-type :type/String}
false lib.types.isa/date-without-time? {:effective-type :type/DateTime :base-type :type/String}
+ true lib.types.isa/date-with-time? {:base-type :type/DateTime}
+ true lib.types.isa/date-with-time? {:effective-type :type/DateTime :base-type :type/String}
+ false lib.types.isa/date-with-time? {:base-type :type/Date}
+ false lib.types.isa/date-with-time? {:base-type :type/String}
+ false lib.types.isa/date-with-time? {:effective-type :type/Date :base-type :type/String}
+
true lib.types.isa/time? {:base-type :type/Time}
true lib.types.isa/time? {:effective-type :type/Time :base-type :type/String}
diff --git a/test/metabase/source_swap/compatibility_test.clj b/test/metabase/source_swap/compatibility_test.clj
index cb31c3f42e0..230d082fd85 100644
--- a/test/metabase/source_swap/compatibility_test.clj
+++ b/test/metabase/source_swap/compatibility_test.clj
@@ -7,51 +7,102 @@
(set! *warn-on-reflection* true)
-(def ^:private mp
- (lib.tu/mock-metadata-provider
- {:database {:id 1 :name "db" :engine :h2}
- :tables [{:id 100 :name "t" :db-id 1}]
- :fields [{:id 1 :table-id 100 :name "PLAIN"
- :base-type :type/Integer :effective-type :type/Integer}
- {:id 2 :table-id 100 :name "DIFFERENT_TYPE"
- :base-type :type/Text :effective-type :type/Text}
- {:id 3 :table-id 100 :name "PK"
- :base-type :type/Integer :effective-type :type/Integer
- :semantic-type :type/PK}
- {:id 4 :table-id 100 :name "FK_A"
- :base-type :type/Integer :effective-type :type/Integer
- :semantic-type :type/FK :fk-target-field-id 999}
- {:id 5 :table-id 100 :name "FK_B"
- :base-type :type/Integer :effective-type :type/Integer
- :semantic-type :type/FK :fk-target-field-id 888}
- {:id 6 :table-id 100 :name "NOT_FK"
- :base-type :type/Integer :effective-type :type/Integer}]}))
-
-(defn- field [id]
+(defn- field [mp id]
(lib.metadata/field mp id))
(deftest ^:parallel column-errors-no-errors-test
- (testing "should return empty errors for matching columns"
- (is (= []
- (source-swap.compatibility/column-errors (field 1) (field 1) :table :table)))))
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "quantity"
+ :base-type :type/Integer :effective-type :type/Integer}
+ {:id 2 :table-id 200 :name "quantity"
+ :base-type :type/Integer :effective-type :type/Integer}]})]
+ (testing "should return empty errors for matching columns"
+ (is (= []
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))
(deftest ^:parallel column-errors-type-mismatch-test
- (testing "should detect column type mismatch"
- (is (= [:column-type-mismatch]
- (source-swap.compatibility/column-errors (field 1) (field 2) :table :table)))))
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "quantity"
+ :base-type :type/Integer :effective-type :type/Integer}
+ {:id 2 :table-id 200 :name "quantity"
+ :base-type :type/Text :effective-type :type/Text}]})]
+ (testing "should detect column type mismatch"
+ (is (= [:column-type-mismatch]
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))
(deftest ^:parallel column-errors-missing-primary-key-test
- (testing "should detect missing primary key for table->table"
- (is (= [:column-type-mismatch :missing-primary-key]
- (source-swap.compatibility/column-errors (field 3) (field 2) :table :table))))
- (testing "should not flag missing primary key for table->card"
- (is (= [:column-type-mismatch]
- (source-swap.compatibility/column-errors (field 3) (field 2) :table :card)))))
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "id"
+ :base-type :type/Integer :effective-type :type/Integer
+ :semantic-type :type/PK}
+ {:id 2 :table-id 200 :name "id"
+ :base-type :type/Text :effective-type :type/Text}]})]
+ (testing "should detect missing primary key for table->table"
+ (is (= [:column-type-mismatch :missing-primary-key]
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))
+ (testing "should not flag missing primary key for table->card"
+ (is (= [:column-type-mismatch]
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :card))))))
(deftest ^:parallel column-errors-foreign-key-test
- (testing "should detect missing foreign key"
- (is (= [:missing-foreign-key]
- (source-swap.compatibility/column-errors (field 4) (field 6) :table :table))))
- (testing "should detect foreign key target mismatch"
- (is (= [:foreign-key-mismatch]
- (source-swap.compatibility/column-errors (field 4) (field 5) :table :table)))))
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}
+ {:id 300 :name "invoices" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "user_id"
+ :base-type :type/Integer :effective-type :type/Integer
+ :semantic-type :type/FK :fk-target-field-id 999}
+ {:id 2 :table-id 200 :name "user_id"
+ :base-type :type/Integer :effective-type :type/Integer
+ :semantic-type :type/FK :fk-target-field-id 888}
+ {:id 3 :table-id 300 :name "user_id"
+ :base-type :type/Integer :effective-type :type/Integer}]})]
+ (testing "should detect missing foreign key"
+ (is (= [:missing-foreign-key]
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 3) :table :table))))
+ (testing "should detect foreign key target mismatch"
+ (is (= [:foreign-key-mismatch]
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))
+
+(deftest ^:parallel column-errors-nil-effective-type-test
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}
+ {:id 300 :name "invoices" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "quantity"
+ :base-type :type/Integer :effective-type :type/Integer}
+ {:id 2 :table-id 200 :name "quantity"
+ :base-type :type/Integer}
+ {:id 3 :table-id 300 :name "quantity"
+ :base-type :type/Text}]})]
+ (testing "same base_type, nil effective_type should be compatible"
+ (is (= []
+ (source-swap.compatibility/column-errors (field mp 2) (field mp 1) :table :table))))
+ (testing "different base_type, nil effective_type should be incompatible"
+ (is (= [:column-type-mismatch]
+ (source-swap.compatibility/column-errors (field mp 2) (field mp 3) :table :table))))))
+
+(deftest ^:parallel column-errors-coerced-effective-type-test
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :engine :h2}
+ :tables [{:id 100 :name "orders" :db-id 1}
+ {:id 200 :name "transactions" :db-id 1}]
+ :fields [{:id 1 :table-id 100 :name "created_at"
+ :base-type :type/DateTime :effective-type :type/DateTime}
+ {:id 2 :table-id 200 :name "created_at"
+ :base-type :type/Text :effective-type :type/DateTime
+ :coercion-strategy :Coercion/ISO8601->DateTime}]})]
+ (testing "text column coerced to DateTime should be compatible with native DateTime"
+ (is (= []
+ (source-swap.compatibility/column-errors (field mp 1) (field mp 2) :table :table))))))