The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[search] better search for CamelCased tables and dbs (#64043)
Oleksandr Solovyovcf5c7bd4c3bd
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 454015f9021..4e131d7e0af 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -51,11 +51,22 @@(defn- searchable-text [m];; For now, we never index the native query content- (->> (:search-terms (search.spec/spec (:model m)))- (keep m)- (map str/trim)- (remove str/blank?)- (str/join " ")))+ (let [search-terms (:search-terms (search.spec/spec (:model m)))+ getter (if (map? search-terms)+ (fn [[k tx]]+ (let [tx (if (true? tx) identity tx)]+ (some-> (get m k) tx)))+ m)+ xf (comp (keep getter)+ (map str/trim)+ (remove str/blank?))]+ (->> (into [] xf search-terms)+ (str/join " "))))++(defn- search-term-columns+ "Extract column names from search-terms spec for SQL query generation"+ [search-terms]+ (if (map? search-terms) (keys search-terms) search-terms))(defn- display-data [m](perf/select-keys m [:name :display_name :description :collection_name]))@@ -107,22 +118,21 @@(defn- spec-index-query*[search-model]- (let [spec (search.spec/spec search-model)- fn-deps (search.spec/collect-fn-attr-req-fields spec)- fn-selects (map (fn [field]- [(keyword (str "this." (name field))) field])- fn-deps)]+ (let [spec (search.spec/spec search-model)+ fn-deps (search.spec/collect-fn-attr-req-fields spec)+ fn-selects (map (fn [field]+ [(keyword (str "this." (name field))) field])+ fn-deps)+ search-terms (set (search-term-columns (:search-terms spec)))](u/remove-nils{:select (search.spec/qualify-columns :this(concat(map (fn [term] [(searchable-value-trim-sql (keyword (str "this." (name term))))term])- (:search-terms spec))+ search-terms)(mapcat (fn [k] (attrs->select-items- (let [search-terms (set (:search-terms spec))]- (->> k- (get spec)- (remove (comp search-terms key))))))+ (->> (get spec k)+ (remove (comp search-terms key)))))[:attrs :render-terms])fn-selects)):from [[(t2/table-name (:model spec)) :this]]diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex 0a65efafaba..2349d833523 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -142,7 +142,9 @@[:visibility [:enum :all :app-user]][:model :keyword][:attrs Attrs]- [:search-terms [:sequential {:min 1} :keyword]]+ [:search-terms [:or+ [:sequential {:min 1} :keyword]+ [:map-of :keyword [:or fn? true?]]]][:render-terms [:map-of NonAttrKey AttrValue]][:where {:optional true} vector?][:bookmark {:optional true} vector?]@@ -215,33 +217,24 @@[[:this (keyword (u/->snake_case_en (name k)))]](find-fields-expr v))))-(defn- find-fields-select-item [x]- (cond- (keyword? x)- (find-fields-kw x)-- (vector? x)- (find-fields-expr (first x))))--(defn- find-fields-top [x]- (cond- (map? x)- (into [] (mapcat find-fields-attr) x)+(defn- find-fields-search [item]+ (let [x (if (map-entry? item) (key item) item)]+ (cond+ (keyword? x)+ (find-fields-kw x)- (sequential? x)- (into [] (mapcat find-fields-select-item) x)-- :else- (throw (ex-info "Unexpected format for fields" {:x x}))))+ (vector? x)+ (find-fields-expr (first x)))))(defn- find-fields"Search within a definition for all the fields referenced on the given table alias."[spec](u/group-by #(nth % 0) #(nth % 1) conj #{}(-> []- (into (mapcat find-fields-top)- ;; Remove the keys with special meanings (should probably switch this to an allowlist rather)- (vals (dissoc spec :name :visibility :native-query :where :joins :bookmark :model)))+ ;; select fields that will influence content+ (into (mapcat find-fields-attr (:attrs spec)))+ (into (mapcat find-fields-search (:search-terms spec)))+ (into (mapcat find-fields-attr (:render-terms spec)))(into (find-fields-expr (:where spec))))))(defn- replace-qualification [expr from to]@@ -408,3 +401,10 @@(let [where (-> (:model/ModelIndexValue (model-hooks)) first :where)](insert-values where :updated {:model_index_id 1 :model_pk 5})))++;;;; indexing helpers++(defn explode-camel-case+ "Transform CamelCase into 'CamelCase Camel Case' so that every word can be searchable"+ [s]+ (str s " " (str/replace s #"([a-z])([A-Z])" "$1 $2")))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex e9c3b919965..ca3aec87180 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -384,7 +384,9 @@:view-count true:created-at true:updated-at true}- :search-terms [:name :display_name :description]+ :search-terms {:name search.spec/explode-camel-case+ :display_name true+ :description true}:render-terms {:initial-sync-status true:table-id :id:table-description :descriptiondiff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex b13491ae766..bab060ee72b 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -581,7 +581,8 @@:database-id false:created-at true:updated-at true}- :search-terms [:name :description]+ :search-terms {:name search.spec/explode-camel-case+ :description true}:where [:= :router_database_id nil]:render-terms {:initial-sync-status true}})diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 72eba5daa20..6bd98c3d27b 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -14,6 +14,47 @@(is (= nil (#'search.ingestion/extract-model-and-id ["dataset" [:and [:= true true] [:= :this.model_id 1901] [:= "Card" "Card"]]])))(is (= nil (#'search.ingestion/extract-model-and-id ["indexed-entity" [:and [:= 26 :this.model_index_id] [:= 38004300 :this.model_pk]]]))))+(deftest searchable-text-test+ (testing "searchable-text with vector format (legacy)"+ (let [spec-fn (constantly {:search-terms [:name :description]})+ record {:model "test"+ :name "Test Name"+ :description "Test Description"+ :other-field "Other Value"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "Test Name Test Description"+ (#'search.ingestion/searchable-text record))))))++ (testing "searchable-text with map format and transforms"+ (let [spec-fn (constantly {:search-terms {:name search.spec/explode-camel-case+ :description true}})+ record {:model "test"+ :name "CamelCaseTest"+ :description "Simple description"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "CamelCaseTest Camel Case Test Simple description"+ (#'search.ingestion/searchable-text record))))))++ (testing "searchable-text filters out blank values"+ (let [spec-fn (constantly {:search-terms [:name :description :empty-field]})+ record {:model "test"+ :name "Test Name"+ :description " " ;; whitespace only+ :empty-field nil}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "Test Name"+ (#'search.ingestion/searchable-text record)))))))++(deftest search-term-columns-test+ (testing "search-term-columns with vector format"+ (is (= #{:name :description}+ (set (#'search.ingestion/search-term-columns [:name :description])))))++ (testing "search-term-columns with map format"+ (is (= #{:name :description}+ (set (#'search.ingestion/search-term-columns {:name identity+ :description nil}))))))+(deftest search-items-count-test(testing "search-items-count returns correct count with various searchable items"(mt/with-empty-h2-app-db!