The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compute longest common substring length with an iterative O(n*m) pass (#80985)
Nicola Mometto190dcf1bd0ac
diff --git a/src/metabase/search/in_place/util.clj b/src/metabase/search/in_place/util.cljindex 9358160dda8..477370889c4 100644--- a/src/metabase/search/in_place/util.clj+++ b/src/metabase/search/in_place/util.clj@@ -1,6 +1,5 @@(ns metabase.search.in-place.util(:require- [clojure.core.memoize :as memoize][clojure.string :as str][metabase.util :as u][metabase.util.malli :as mu]))@@ -21,29 +20,16 @@(filter seq(str/split query #"\s+")))-(def largest-common-subseq-length+(defn largest-common-subseq-length"Given two lists (and an equality test), return the length of the longest overlapping subsequence.(largest-common-subseq-length = [1 2 3 :this :part :will :not :be :relevant][:not :counted 1 2 3 :also :not :counted])- ;; => 3"- (memoize/fifo- (fn- ([eq xs ys]- (largest-common-subseq-length eq xs ys 0))- ([eq xs ys tally]- (if (or (zero? (count xs))- (zero? (count ys)))- tally- (max- (if (eq (first xs)- (first ys))- (largest-common-subseq-length eq (rest xs) (rest ys) (inc tally))- tally)- (largest-common-subseq-length eq xs (rest ys) 0)- (largest-common-subseq-length eq (rest xs) ys 0)))))- ;; Uses O(n*m) space (the lengths of the two lists) with k≤2, so napkin math suggests this gives us caching for at- ;; least a 31*31 search (or 50*20, etc) which sounds like more than enough. Memory is cheap and the items are- ;; small, so we may as well skew high.- ;; As a precaution, the scorer that uses this limits the number of tokens (see the `take` call below)- :fifo/threshold 2000))+ ;; => 3++ Iterative O(n*m) DP: each row holds the length of the common run ending at the current `x` and each `y`."+ [eq xs ys]+ (let [next-row (fn [prev-row x]+ (into [0] (map-indexed (fn [j y] (if (eq x y) (inc (nth prev-row j 0)) 0))) ys))+ rows (reductions next-row [] xs)]+ (transduce cat max 0 rows)))diff --git a/test/metabase/search/in_place/util_test.clj b/test/metabase/search/in_place/util_test.cljindex 5f4f1fe8b7c..80f610e52be 100644--- a/test/metabase/search/in_place/util_test.clj+++ b/test/metabase/search/in_place/util_test.clj@@ -1,5 +1,6 @@(ns metabase.search.in-place.util-test(:require+ [clojure.string :as str][clojure.test :refer :all][metabase.search.in-place.util :as search.util]))@@ -33,4 +34,40 @@(map str '(here is some fillerthis social bird lives in small flocks in lowland rainforests in countries such as costa ricait flies short distances between trees toucans rest in holes in trees- here is some more filler))))))))+ here is some more filler))))))+ (testing "empty inputs"+ (is (= 0 (subseq-length [] [])))+ (is (= 0 (subseq-length [] [1 2 3])))+ (is (= 0 (subseq-length [1 2 3] []))))+ (testing "custom equality predicate"+ (let [substring-match? (fn [needle haystack] (str/includes? haystack needle))]+ (is (= 2+ (search.util/largest-common-subseq-length substring-match?+ ["gar" "pa"]+ ["foo" "garden" "path"])))))))++(deftest ^:parallel largest-common-subseq-length-large-grid-test+ (let [subseq-length (partial search.util/largest-common-subseq-length =)]+ (testing "a 30x30 fully-matching grid computes promptly with the correct length"+ (is (= 30 (subseq-length (repeat 30 :x) (repeat 30 :x)))))+ (testing "no matches at all (every cell is a dead end)"+ (is (= 0 (subseq-length (range 200) (range 200 400)))))+ (testing "alternating tokens: many short runs that never chain"+ (is (= 1 (subseq-length (take 200 (cycle [:a :b])) (repeat 200 :a)))))+ (testing "a single mismatch in the middle halves the run"+ (is (= 100 (subseq-length (repeat 201 :x) (concat (repeat 100 :x) [:y] (repeat 100 :x))))))+ (testing "asymmetric inputs"+ (is (= 1 (subseq-length [:x] (repeat 5000 :x))))+ (is (= 1 (subseq-length (repeat 5000 :x) [:x]))))+ (testing "well above the 30-token cap the scorer applies"+ (is (= 500 (subseq-length (range 500) (range 500)))))))++(deftest ^:parallel largest-common-subseq-length-eq-call-count-test+ (testing "eq is invoked exactly once per cell of the n*m grid, regardless of match density"+ (doseq [[xs ys] [[(range 40) (range 25)]+ [(repeat 40 :x) (repeat 25 :x)]+ [(range 40) (range 100 125)]]]+ (let [calls (atom 0)+ eq (fn [a b] (swap! calls inc) (= a b))]+ (search.util/largest-common-subseq-length eq xs ys)+ (is (= (* 40 25) @calls))))))