Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix clickhouse uuid filtering (#66852)

Riley Thompson653d9ab52f90
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
index 67d5e0e4733..fdadfffe018 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
@@ -15,7 +15,8 @@
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
- [metabase.util.performance :refer [some]])
+ [metabase.util.performance :refer [some]]
+ [metabase.util.string :as string])
(:import
[java.net Inet4Address Inet6Address]
[java.sql ResultSet ResultSetMetaData Types]
@@ -357,40 +358,56 @@
:type/IPAddress [:'toIPv4 value]
(sql.qp/->honeysql driver value)))))
+(defn- text-val? [value]
+ (let [[qual valuevalue fieldinfo] value]
+ (and (isa? qual :value)
+ (isa? (:base_type fieldinfo) :type/Text)
+ (nil? valuevalue))))
+
+(defn- uuid-comp? [field value]
+ (let [[qual valuevalue fieldinfo] value]
+ (and (isa? qual :value)
+ (isa? (:base_type fieldinfo) :type/UUID)
+ (isa? (:base-type (nth field 2)) :type/UUID)
+ (string? valuevalue))))
+
(defmethod sql.qp/->honeysql [:clickhouse :=]
[driver [op field value]]
- (let [[qual valuevalue fieldinfo] value
- hsql-field (sql.qp/->honeysql driver field)
+ (let [hsql-field (sql.qp/->honeysql driver field)
hsql-value (sql.qp/->honeysql driver value)]
(cond
- (and (isa? qual :value)
- (isa? (:base_type fieldinfo) :type/Text)
- (nil? valuevalue))
+ (text-val? value)
[:or
[:= hsql-field hsql-value]
[:= [:'empty hsql-field] 1]]
;; UUID fields can be compared directly with strings in ClickHouse.
- (and (isa? qual :value)
- (isa? (:base_type fieldinfo) :type/UUID)
- (isa? (:base-type (nth field 2)) :type/UUID)
- (string? valuevalue))
- [:= hsql-field hsql-value]
+ ;; If the string is not a valid UUID (ie due to is-empty desugaring),
+ ;; then direct comparison will cause an error, so just return false
+ (uuid-comp? field value)
+ (if (string/valid-uuid? hsql-value)
+ [:= hsql-field hsql-value]
+ false)
:else ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value]))))
(defmethod sql.qp/->honeysql [:clickhouse :!=]
[driver [op field value]]
- (let [[qual valuevalue fieldinfo] value
- hsql-field (sql.qp/->honeysql driver field)
+ (let [hsql-field (sql.qp/->honeysql driver field)
hsql-value (sql.qp/->honeysql driver value)]
- (if (and (isa? qual :value)
- (isa? (:base_type fieldinfo) :type/Text)
- (nil? valuevalue))
+ (cond
+ (text-val? value)
[:and
[:!= hsql-field hsql-value]
[:= [:'notEmpty hsql-field] 1]]
- ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value]))))
+
+ (uuid-comp? field value)
+ (if (string/valid-uuid? hsql-value)
+ [:or [:!= hsql-field hsql-value]
+ [:isNull hsql-field]]
+ true)
+
+ :else ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value]))))
;; I do not know why the tests expect nil counts for empty results
;; but that's how it is :-)
@@ -417,7 +434,9 @@
(defn- clickhouse-string-fn
[fn-name field value options]
- (let [hsql-field (sql.qp/->honeysql :clickhouse field)
+ (let [[_ _ {:keys [base-type]}] field
+ hsql-field (cond->> (sql.qp/->honeysql :clickhouse field)
+ (= base-type :type/UUID) (conj [:'toString]))
hsql-value (sql.qp/->honeysql :clickhouse value)]
(if (get options :case-sensitive true)
[fn-name hsql-field hsql-value]
@@ -439,7 +458,9 @@
(defmethod sql.qp/->honeysql [:clickhouse :contains]
[_ [_ field value options]]
- (let [hsql-field (sql.qp/->honeysql :clickhouse field)
+ (let [[_ _ {:keys [base-type]}] field
+ hsql-field (cond->> (sql.qp/->honeysql :clickhouse field)
+ (= base-type :type/UUID) (conj [:'toString]))
hsql-value (sql.qp/->honeysql :clickhouse value)
position-fn (if (get options :case-sensitive true)
:'positionUTF8
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
index fc57ca45e4e..db560c6ff1c 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
[metabase.test :as mt]
[metabase.test.data.clickhouse :as ctd]))
@@ -337,22 +338,72 @@
array_of_tuples_test
{})))))))))
-(deftest ^:parallel uuid-literals-test
+(deftest ^:parallel uuid-filtering-test
(mt/test-driver :clickhouse
(mt/dataset
(mt/dataset-definition
- "metabase_tests_uuids_nullable"
- [["uuids_nullable"
+ "nullable_uuids_dataset"
+ [["nullable_uuids"
[{:field-name "uuid1", :base-type {:native "Nullable(UUID)"}}]
- [[#uuid "550e8400-e29b-41d4-a716-446655440000"]
+ [[#uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]
+ [#uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
[nil]]]])
- (let [mp (mt/metadata-provider)]
- (is (= [[1 #uuid "550e8400-e29b-41d4-a716-446655440000"]]
- (-> (lib/query mp (lib.metadata/table mp (mt/id :uuids_nullable)))
- (lib/filter (lib/= (lib.metadata/field mp (mt/id :uuids_nullable :uuid1))
- "550e8400-e29b-41d4-a716-446655440000"))
- mt/process-query
- mt/rows)))))))
+ (let [mp (mt/metadata-provider)
+ uuid-field (lib.metadata/field mp (mt/id :nullable_uuids :uuid1))
+ filter-rows (fn [filter]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :nullable_uuids)))
+ (lib/filter filter)
+ mt/process-query
+ mt/rows))]
+ (testing "can filter nullable uuids with equals and not equals"
+ (are [filter-fn exp-rows]
+ (= exp-rows (filter-rows (filter-fn uuid-field "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")))
+ lib/= [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ lib/!= [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
+ [3 nil]]))
+ (testing "can filter nullable uuids with empty and not empty"
+ (are [filter-fn exp-rows]
+ (= exp-rows (filter-rows (filter-fn uuid-field)))
+ lib/is-empty [[3 nil]]
+ lib/not-empty [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]
+ [2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]))
+ (testing "can filter nullable uuids with contains"
+ (are [filter-str case-sensitive exp-rows]
+ (= exp-rows (filter-rows (-> (lib/contains uuid-field filter-str)
+ (lib.options/update-options assoc :case-sensitive case-sensitive))))
+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "AAA" true []
+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))
+ (testing "can filter nullable uuids with does not contain"
+ (are [filter-str case-sensitive exp-rows]
+ (= exp-rows (filter-rows (-> (lib/does-not-contain uuid-field filter-str)
+ (lib.options/update-options assoc :case-sensitive case-sensitive))))
+ "aaa" true [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
+ [3 nil]]
+ "aaa" false [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
+ [3 nil]]
+ "AAA" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]
+ [2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
+ [3 nil]]
+ "AAA" false [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]
+ [3 nil]]))
+ (testing "can filter nullable uuids with starts with"
+ (are [filter-str case-sensitive exp-rows]
+ (= exp-rows (filter-rows (-> (lib/starts-with uuid-field filter-str)
+ (lib.options/update-options assoc :case-sensitive case-sensitive))))
+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "AAA" true []
+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))
+ (testing "can filter nullable uuids with ends with"
+ (are [filter-str case-sensitive exp-rows]
+ (= exp-rows (filter-rows (-> (lib/ends-with uuid-field filter-str)
+ (lib.options/update-options assoc :case-sensitive case-sensitive))))
+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]
+ "AAA" true []
+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))))))
#_(deftest ^:parallel clickhouse-array-of-uuids
(mt/test-driver :clickhouse