Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[search] better search for CamelCased tables and dbs (#64043)

Oleksandr Solovyovcf5c7bd4c3bd
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 454015f9021..4e131d7e0af 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -51,11 +51,22 @@
(defn- searchable-text [m]
;; For now, we never index the native query content
- (->> (:search-terms (search.spec/spec (:model m)))
- (keep m)
- (map str/trim)
- (remove str/blank?)
- (str/join " ")))
+ (let [search-terms (:search-terms (search.spec/spec (:model m)))
+ getter (if (map? search-terms)
+ (fn [[k tx]]
+ (let [tx (if (true? tx) identity tx)]
+ (some-> (get m k) tx)))
+ m)
+ xf (comp (keep getter)
+ (map str/trim)
+ (remove str/blank?))]
+ (->> (into [] xf search-terms)
+ (str/join " "))))
+
+(defn- search-term-columns
+ "Extract column names from search-terms spec for SQL query generation"
+ [search-terms]
+ (if (map? search-terms) (keys search-terms) search-terms))
(defn- display-data [m]
(perf/select-keys m [:name :display_name :description :collection_name]))
@@ -107,22 +118,21 @@
(defn- spec-index-query*
[search-model]
- (let [spec (search.spec/spec search-model)
- fn-deps (search.spec/collect-fn-attr-req-fields spec)
- fn-selects (map (fn [field]
- [(keyword (str "this." (name field))) field])
- fn-deps)]
+ (let [spec (search.spec/spec search-model)
+ fn-deps (search.spec/collect-fn-attr-req-fields spec)
+ fn-selects (map (fn [field]
+ [(keyword (str "this." (name field))) field])
+ fn-deps)
+ search-terms (set (search-term-columns (:search-terms spec)))]
(u/remove-nils
{:select (search.spec/qualify-columns :this
(concat
(map (fn [term] [(searchable-value-trim-sql (keyword (str "this." (name term))))
term])
- (:search-terms spec))
+ search-terms)
(mapcat (fn [k] (attrs->select-items
- (let [search-terms (set (:search-terms spec))]
- (->> k
- (get spec)
- (remove (comp search-terms key))))))
+ (->> (get spec k)
+ (remove (comp search-terms key)))))
[:attrs :render-terms])
fn-selects))
:from [[(t2/table-name (:model spec)) :this]]
diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.clj
index 0a65efafaba..2349d833523 100644
--- a/src/metabase/search/spec.clj
+++ b/src/metabase/search/spec.clj
@@ -142,7 +142,9 @@
[:visibility [:enum :all :app-user]]
[:model :keyword]
[:attrs Attrs]
- [:search-terms [:sequential {:min 1} :keyword]]
+ [:search-terms [:or
+ [:sequential {:min 1} :keyword]
+ [:map-of :keyword [:or fn? true?]]]]
[:render-terms [:map-of NonAttrKey AttrValue]]
[:where {:optional true} vector?]
[:bookmark {:optional true} vector?]
@@ -215,33 +217,24 @@
[[:this (keyword (u/->snake_case_en (name k)))]]
(find-fields-expr v))))
-(defn- find-fields-select-item [x]
- (cond
- (keyword? x)
- (find-fields-kw x)
-
- (vector? x)
- (find-fields-expr (first x))))
-
-(defn- find-fields-top [x]
- (cond
- (map? x)
- (into [] (mapcat find-fields-attr) x)
+(defn- find-fields-search [item]
+ (let [x (if (map-entry? item) (key item) item)]
+ (cond
+ (keyword? x)
+ (find-fields-kw x)
- (sequential? x)
- (into [] (mapcat find-fields-select-item) x)
-
- :else
- (throw (ex-info "Unexpected format for fields" {:x x}))))
+ (vector? x)
+ (find-fields-expr (first x)))))
(defn- find-fields
"Search within a definition for all the fields referenced on the given table alias."
[spec]
(u/group-by #(nth % 0) #(nth % 1) conj #{}
(-> []
- (into (mapcat find-fields-top)
- ;; Remove the keys with special meanings (should probably switch this to an allowlist rather)
- (vals (dissoc spec :name :visibility :native-query :where :joins :bookmark :model)))
+ ;; select fields that will influence content
+ (into (mapcat find-fields-attr (:attrs spec)))
+ (into (mapcat find-fields-search (:search-terms spec)))
+ (into (mapcat find-fields-attr (:render-terms spec)))
(into (find-fields-expr (:where spec))))))
(defn- replace-qualification [expr from to]
@@ -408,3 +401,10 @@
(let [where (-> (:model/ModelIndexValue (model-hooks)) first :where)]
(insert-values where :updated {:model_index_id 1 :model_pk 5})))
+
+;;;; indexing helpers
+
+(defn explode-camel-case
+ "Transform CamelCase into 'CamelCase Camel Case' so that every word can be searchable"
+ [s]
+ (str s " " (str/replace s #"([a-z])([A-Z])" "$1 $2")))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index e9c3b919965..ca3aec87180 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -384,7 +384,9 @@
:view-count true
:created-at true
:updated-at true}
- :search-terms [:name :display_name :description]
+ :search-terms {:name search.spec/explode-camel-case
+ :display_name true
+ :description true}
:render-terms {:initial-sync-status true
:table-id :id
:table-description :description
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index b13491ae766..bab060ee72b 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -581,7 +581,8 @@
:database-id false
:created-at true
:updated-at true}
- :search-terms [:name :description]
+ :search-terms {:name search.spec/explode-camel-case
+ :description true}
:where [:= :router_database_id nil]
:render-terms {:initial-sync-status true}})
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 72eba5daa20..6bd98c3d27b 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -14,6 +14,47 @@
(is (= nil (#'search.ingestion/extract-model-and-id ["dataset" [:and [:= true true] [:= :this.model_id 1901] [:= "Card" "Card"]]])))
(is (= nil (#'search.ingestion/extract-model-and-id ["indexed-entity" [:and [:= 26 :this.model_index_id] [:= 38004300 :this.model_pk]]]))))
+(deftest searchable-text-test
+ (testing "searchable-text with vector format (legacy)"
+ (let [spec-fn (constantly {:search-terms [:name :description]})
+ record {:model "test"
+ :name "Test Name"
+ :description "Test Description"
+ :other-field "Other Value"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "Test Name Test Description"
+ (#'search.ingestion/searchable-text record))))))
+
+ (testing "searchable-text with map format and transforms"
+ (let [spec-fn (constantly {:search-terms {:name search.spec/explode-camel-case
+ :description true}})
+ record {:model "test"
+ :name "CamelCaseTest"
+ :description "Simple description"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "CamelCaseTest Camel Case Test Simple description"
+ (#'search.ingestion/searchable-text record))))))
+
+ (testing "searchable-text filters out blank values"
+ (let [spec-fn (constantly {:search-terms [:name :description :empty-field]})
+ record {:model "test"
+ :name "Test Name"
+ :description " " ;; whitespace only
+ :empty-field nil}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "Test Name"
+ (#'search.ingestion/searchable-text record)))))))
+
+(deftest search-term-columns-test
+ (testing "search-term-columns with vector format"
+ (is (= #{:name :description}
+ (set (#'search.ingestion/search-term-columns [:name :description])))))
+
+ (testing "search-term-columns with map format"
+ (is (= #{:name :description}
+ (set (#'search.ingestion/search-term-columns {:name identity
+ :description nil}))))))
+
(deftest search-items-count-test
(testing "search-items-count returns correct count with various searchable items"
(mt/with-empty-h2-app-db!