Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Minor fixes around semantic search (#65654)

Chris Truter3c92dcd0715c
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index e059f78c7ed..08e8b96faf1 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -58,21 +58,26 @@
final-count threshold raw-count fallback)
(analytics/inc! :metabase-search/semantic-fallback-triggered {:fallback-engine fallback})
(analytics/observe! :metabase-search/semantic-results-before-fallback final-count)
- (let [fallback-results (try
+
+ (when (some-> (:offset-int search-ctx) pos?)
+ (log/warn "Using an offset with semantic search will produce strange results, e.g. missing expected results, or duplicating them across pages"))
+
+ (let [total-limit (semantic.settings/semantic-search-results-limit)
+ fallback-results (try
(cond->> (search.engine/results (assoc search-ctx :search-engine fallback))
;; The in-place engine returns a reducible (but not seqable) result that needs to
;; be realized before we concat and dedup with the semantic engine results.
(= :search.engine/in-place fallback)
(into [] (comp (map t2.realize/realize)
- (take (- (semantic.settings/semantic-search-results-limit) final-count)))))
+ (take total-limit))))
(catch Throwable t
(log/warn t "Semantic search fallback errored, ignoring")
[]))
- _ (analytics/observe! :metabase-search/semantic-fallback-results-usage
- (count fallback-results))
+ fallback-results (take total-limit fallback-results)
+ _ (analytics/observe! :metabase-search/semantic-fallback-results-usage (count fallback-results))
combined-results (concat results fallback-results)
deduped-results (m/distinct-by (juxt :model :id) combined-results)]
- (take (semantic.settings/semantic-search-results-limit) deduped-results)))))
+ (take total-limit deduped-results)))))
(catch Exception e
(log/error e "Error executing semantic search")
(throw (ex-info "Error executing semantic search" {:type :semantic-search-error} e)))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
index 8191dd79e35..13468e530e2 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
@@ -71,8 +71,8 @@
(defn base-scorers
"The default constituents of the search ranking scores."
- [index-table {:keys [search-string limit-int] :as search-ctx}]
- (if (and limit-int (zero? limit-int))
+ [index-table {:keys [search-string] :as search-ctx}]
+ (if (search.scoring/no-scoring-required? search-ctx)
{:model [:inline 1]}
;; NOTE: we calculate scores even if the weight is zero, so that it's easy to consider how we could affect any
;; given set of results. At some point, we should optimize away the irrelevant scores for any given context.
@@ -188,8 +188,8 @@
(defn appdb-scorers
"The appdb-based scorers for search ranking results. Like `base-scorers`, but for scorers that need to query the appdb."
- [{:keys [limit-int] :as search-ctx}]
- (when-not (and limit-int (zero? limit-int))
+ [search-ctx]
+ (when-not (search.scoring/no-scoring-required? search-ctx)
{:bookmarked search.scoring/bookmark-score-expr
:user-recency (search.scoring/inverse-duration
(search.scoring/user-recency-expr search-ctx) [:now] search.config/stale-time-in-days)}))
diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.clj
index cb2807afc63..2fbfa86065c 100644
--- a/src/metabase/search/appdb/scoring.clj
+++ b/src/metabase/search/appdb/scoring.clj
@@ -41,8 +41,8 @@
(defn base-scorers
"The default constituents of the search ranking scores."
- [{:keys [search-string limit-int] :as search-ctx}]
- (if (and limit-int (zero? limit-int))
+ [{:keys [search-string] :as search-ctx}]
+ (if (search.scoring/no-scoring-required? search-ctx)
{:model [:inline 1]}
;; NOTE: we calculate scores even if the weight is zero, so that it's easy to consider how we could affect any
;; given set of results. At some point, we should optimize away the irrelevant scores for any given context.
diff --git a/src/metabase/search/scoring.clj b/src/metabase/search/scoring.clj
index be4a2c5e2cb..5ecc47a4348 100644
--- a/src/metabase/search/scoring.clj
+++ b/src/metabase/search/scoring.clj
@@ -172,3 +172,8 @@
:weight weight
:contribution (* weight score)}))
scorers))
+
+(defn no-scoring-required?
+ "Scoring is unnecessary when we are not returning any results, e.g. counting potential results"
+ [{:keys [limit-int]}]
+ (and limit-int (zero? limit-int)))