The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix clickhouse uuid filtering (#66852)
Riley Thompson653d9ab52f90
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.cljindex 67d5e0e4733..fdadfffe018 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj@@ -15,7 +15,8 @@[metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x]- [metabase.util.performance :refer [some]])+ [metabase.util.performance :refer [some]]+ [metabase.util.string :as string])(:import[java.net Inet4Address Inet6Address][java.sql ResultSet ResultSetMetaData Types]@@ -357,40 +358,56 @@:type/IPAddress [:'toIPv4 value](sql.qp/->honeysql driver value)))))+(defn- text-val? [value]+ (let [[qual valuevalue fieldinfo] value]+ (and (isa? qual :value)+ (isa? (:base_type fieldinfo) :type/Text)+ (nil? valuevalue))))++(defn- uuid-comp? [field value]+ (let [[qual valuevalue fieldinfo] value]+ (and (isa? qual :value)+ (isa? (:base_type fieldinfo) :type/UUID)+ (isa? (:base-type (nth field 2)) :type/UUID)+ (string? valuevalue))))+(defmethod sql.qp/->honeysql [:clickhouse :=][driver [op field value]]- (let [[qual valuevalue fieldinfo] value- hsql-field (sql.qp/->honeysql driver field)+ (let [hsql-field (sql.qp/->honeysql driver field)hsql-value (sql.qp/->honeysql driver value)](cond- (and (isa? qual :value)- (isa? (:base_type fieldinfo) :type/Text)- (nil? valuevalue))+ (text-val? value)[:or[:= hsql-field hsql-value][:= [:'empty hsql-field] 1]];; UUID fields can be compared directly with strings in ClickHouse.- (and (isa? qual :value)- (isa? (:base_type fieldinfo) :type/UUID)- (isa? (:base-type (nth field 2)) :type/UUID)- (string? valuevalue))- [:= hsql-field hsql-value]+ ;; If the string is not a valid UUID (ie due to is-empty desugaring),+ ;; then direct comparison will cause an error, so just return false+ (uuid-comp? field value)+ (if (string/valid-uuid? hsql-value)+ [:= hsql-field hsql-value]+ false):else ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value]))))(defmethod sql.qp/->honeysql [:clickhouse :!=][driver [op field value]]- (let [[qual valuevalue fieldinfo] value- hsql-field (sql.qp/->honeysql driver field)+ (let [hsql-field (sql.qp/->honeysql driver field)hsql-value (sql.qp/->honeysql driver value)]- (if (and (isa? qual :value)- (isa? (:base_type fieldinfo) :type/Text)- (nil? valuevalue))+ (cond+ (text-val? value)[:and[:!= hsql-field hsql-value][:= [:'notEmpty hsql-field] 1]]- ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value]))))++ (uuid-comp? field value)+ (if (string/valid-uuid? hsql-value)+ [:or [:!= hsql-field hsql-value]+ [:isNull hsql-field]]+ true)++ :else ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value]))));; I do not know why the tests expect nil counts for empty results;; but that's how it is :-)@@ -417,7 +434,9 @@(defn- clickhouse-string-fn[fn-name field value options]- (let [hsql-field (sql.qp/->honeysql :clickhouse field)+ (let [[_ _ {:keys [base-type]}] field+ hsql-field (cond->> (sql.qp/->honeysql :clickhouse field)+ (= base-type :type/UUID) (conj [:'toString]))hsql-value (sql.qp/->honeysql :clickhouse value)](if (get options :case-sensitive true)[fn-name hsql-field hsql-value]@@ -439,7 +458,9 @@(defmethod sql.qp/->honeysql [:clickhouse :contains][_ [_ field value options]]- (let [hsql-field (sql.qp/->honeysql :clickhouse field)+ (let [[_ _ {:keys [base-type]}] field+ hsql-field (cond->> (sql.qp/->honeysql :clickhouse field)+ (= base-type :type/UUID) (conj [:'toString]))hsql-value (sql.qp/->honeysql :clickhouse value)position-fn (if (get options :case-sensitive true):'positionUTF8diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.cljindex fc57ca45e4e..db560c6ff1c 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options][metabase.test :as mt][metabase.test.data.clickhouse :as ctd]))@@ -337,22 +338,72 @@array_of_tuples_test{})))))))))-(deftest ^:parallel uuid-literals-test+(deftest ^:parallel uuid-filtering-test(mt/test-driver :clickhouse(mt/dataset(mt/dataset-definition- "metabase_tests_uuids_nullable"- [["uuids_nullable"+ "nullable_uuids_dataset"+ [["nullable_uuids"[{:field-name "uuid1", :base-type {:native "Nullable(UUID)"}}]- [[#uuid "550e8400-e29b-41d4-a716-446655440000"]+ [[#uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]+ [#uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"][nil]]]])- (let [mp (mt/metadata-provider)]- (is (= [[1 #uuid "550e8400-e29b-41d4-a716-446655440000"]]- (-> (lib/query mp (lib.metadata/table mp (mt/id :uuids_nullable)))- (lib/filter (lib/= (lib.metadata/field mp (mt/id :uuids_nullable :uuid1))- "550e8400-e29b-41d4-a716-446655440000"))- mt/process-query- mt/rows)))))))+ (let [mp (mt/metadata-provider)+ uuid-field (lib.metadata/field mp (mt/id :nullable_uuids :uuid1))+ filter-rows (fn [filter]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :nullable_uuids)))+ (lib/filter filter)+ mt/process-query+ mt/rows))]+ (testing "can filter nullable uuids with equals and not equals"+ (are [filter-fn exp-rows]+ (= exp-rows (filter-rows (filter-fn uuid-field "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")))+ lib/= [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ lib/!= [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]+ [3 nil]]))+ (testing "can filter nullable uuids with empty and not empty"+ (are [filter-fn exp-rows]+ (= exp-rows (filter-rows (filter-fn uuid-field)))+ lib/is-empty [[3 nil]]+ lib/not-empty [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]+ [2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]))+ (testing "can filter nullable uuids with contains"+ (are [filter-str case-sensitive exp-rows]+ (= exp-rows (filter-rows (-> (lib/contains uuid-field filter-str)+ (lib.options/update-options assoc :case-sensitive case-sensitive))))+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "AAA" true []+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))+ (testing "can filter nullable uuids with does not contain"+ (are [filter-str case-sensitive exp-rows]+ (= exp-rows (filter-rows (-> (lib/does-not-contain uuid-field filter-str)+ (lib.options/update-options assoc :case-sensitive case-sensitive))))+ "aaa" true [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]+ [3 nil]]+ "aaa" false [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]+ [3 nil]]+ "AAA" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]+ [2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]+ [3 nil]]+ "AAA" false [[2 #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]+ [3 nil]]))+ (testing "can filter nullable uuids with starts with"+ (are [filter-str case-sensitive exp-rows]+ (= exp-rows (filter-rows (-> (lib/starts-with uuid-field filter-str)+ (lib.options/update-options assoc :case-sensitive case-sensitive))))+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "AAA" true []+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))+ (testing "can filter nullable uuids with ends with"+ (are [filter-str case-sensitive exp-rows]+ (= exp-rows (filter-rows (-> (lib/ends-with uuid-field filter-str)+ (lib.options/update-options assoc :case-sensitive case-sensitive))))+ "aaa" true [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "aaa" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]+ "AAA" true []+ "AAA" false [[1 #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]]))))))#_(deftest ^:parallel clickhouse-array-of-uuids(mt/test-driver :clickhouse