The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix ordered hierarchy ordering and validation (#80720)
Chris Trutera5bd647dc48d
diff --git a/src/metabase/util/ordered_hierarchy.clj b/src/metabase/util/ordered_hierarchy.cljindex ee1297352d2..026fecd0ce7 100644--- a/src/metabase/util/ordered_hierarchy.clj+++ b/src/metabase/util/ordered_hierarchy.clj@@ -2,88 +2,125 @@(ns metabase.util.ordered-hierarchy(:refer-clojure :exclude [ancestors derive descendants make-hierarchy parents])(:require- [flatland.ordered.set :refer [ordered-set]]- [medley.core :as m]))+ [flatland.ordered.map :refer [ordered-map]]+ [flatland.ordered.set :refer [ordered-set]]))-(declare derive)+(declare calculate-derived-fields derive)++(defn- known-tag?+ [h tag]+ (or (contains? (:parents h) tag)+ (contains? (:children h) tag)))++(defn- check-first-occurrence!+ [h [tag & children :as basis]]+ (when (and (known-tag? h tag) (seq children))+ (throw (ex-info (format "Children of %s may only be listed at its first occurrence" tag)+ {:basis basis :tag tag}))))++(defn- register-parent+ [h parent]+ (if (contains? (:children h) parent)+ h+ (assoc h :children (assoc (:children h) parent nil))))(defn- derive-children- [h [parent & children]]- (reduce (fn [h child]- (cond- (keyword? child) (derive h child parent)- (vector? child) (let [grandchild (first child)]- (assert (not (and (contains? (:parents h) grandchild) (rest child)))- (format "You may only list a %s's children at its first occurrence"- grandchild))- (derive-children (derive h grandchild parent)- child))))- h- children))+ [h [parent & children :as basis]]+ (when-not (keyword? parent)+ (throw (ex-info "Hierarchy vectors must begin with a keyword tag"+ {:basis basis})))+ (let [h (register-parent h parent)]+ (if (seq children)+ (reduce (fn [h child]+ (cond+ (keyword? child)+ (derive h child parent)++ (vector? child)+ (let [grandchild (first child)]+ (when-not (keyword? grandchild)+ (throw (ex-info "Hierarchy vectors must begin with a keyword tag"+ {:basis child})))+ (check-first-occurrence! h child)+ (derive-children (derive h grandchild parent) child))++ :else+ (throw (ex-info "Hierarchy children must be keywords or vectors"+ {:basis basis :child child}))))+ h+ children)+ (calculate-derived-fields h))))(defn- derive-basis [h basis]- (cond (vector? basis) (derive-children h basis)- :else (throw (ex-info (str "Unsupported type for ordered-hierarchy: " (type basis))- {:h h :basis basis}))))+ (when-not (vector? basis)+ (throw (ex-info (str "Hierarchy basis must be a vector, got " (type basis))+ {:basis basis})))+ (check-first-occurrence! h basis)+ (derive-children h basis))(defn make-hierarchy- "Similar to [[clojure.core/make-hierarchy]], but the returned hierarchy has well-defined orderings for its sets.+ "Creates a hierarchy whose sets have deterministic iteration order.- Can take arguments to be treated as roots, defined using hiccup syntax.+ Each optional basis uses Hiccup-style syntax: `[parent child [child grandchild ...] ...]`.- !! WARNING !!- Using [[clojure.core/derive]] with this will corrupt the ordering - you must use the implementation from this ns."+ Always use [[derive]] to extend the result.+ [[clojure.core/derive]] does not maintain this namespace's custom fields or metadata."([](-> (clojure.core/make-hierarchy)- (with-meta {::ordered? true})))+ (assoc :children (ordered-map)+ :sorted-tags (ordered-set))+ (vary-meta assoc ::ordered? true)))([& bases](reduce derive-basis (make-hierarchy) bases)))(defn ancestors- "Returns the immediate and indirect parents of tag, as established via derive. Earlier derivations are shown first.- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."+ "Returns the immediate and indirect parents of `tag` in leaves-to-roots topological order.++ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."[h tag](clojure.core/ancestors h tag))(defn descendants- "Returns the immediate and indirect children of tag, as established via derive. Earlier derivations are shown later.- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."+ "Returns the immediate and indirect children of `tag` in reverse topological order.++ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."[h tag](clojure.core/descendants h tag))(defn parents- "Returns the immediate parents of tag, as established via derive. Earlier derivations are shown first.- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."+ "Returns the immediate parents of `tag` in derivation order.++ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."[h tag](clojure.core/parents h tag))(defn children- "Returns the immediate children of tag, as established via derive. Later derivations are shown first."+ "Returns the immediate children of `tag` in reverse derivation order."[h tag]- (-> h :children tag))+ (get-in h [:children tag]))-(defn- toposort-visit [g state n]+(defn- toposort-visit [graph state node](let [{:keys [processing processed]} state]- (when (contains? processing n)+ (when (contains? processing node)(throw (ex-info "Cycle in graph" {:cause ::cyclic-graph- :cycle-node n+ :cycle-node node:state state- :graph g})))- (if (contains? processed n)+ :graph graph})))+ (if (contains? processed node)state- (let [children (get g n)- init-state (update state :processing conj n)- post-state (reduce (partial toposort-visit g)+ (let [children (get graph node)+ init-state (update state :processing conj node)+ post-state (reduce (partial toposort-visit graph)init-state- ;; Iterate in reverse as that gives us oldest to newest+ ;; Children are stored newest-first; visit them in derivation order.(reverse children))](-> post-state- (update :processed conj n)- (update :processing disj n))))))+ (update :processed conj node)+ (update :processing disj node))))))(defn- toposort- "Use Trajan's DPS method to return an ordered set of nodes where every node is preceded by all its children, and- those children are likewise enumerated in the order they were registered as children of that node."+ "Performs a depth-first topological sort.+ Children precede their parents, and sibling ties follow derivation order."[roots graph](->> roots(reduce (partial toposort-visit graph)@@ -91,18 +128,22 @@:processed (ordered-set)}):processed))-(defn sorted-tags- "An ordered set of all tags within the hierarchy, topologically sorted from the leaves up, and following the edges- according to the order they were added in."+(defn- calculate-sorted-tags[h](let [roots (reduce disj(into (ordered-set) (keys (:children h)))(keys (:parents h)))](toposort roots (:children h))))+(defn sorted-tags+ "Returns all tags in leaves-to-roots topological order, using derivation order to break ties."+ [h]+ (or (:sorted-tags h)+ (calculate-sorted-tags h)))+(defn- calculate-derived-fields [h]- (let [ts (sorted-tags h)- re-sort (fn [m sorted] (m/map-vals #(into (ordered-set) (filter % sorted)) m))+ (let [ts (calculate-sorted-tags h)+ re-sort (fn [m order] (update-vals m #(into (ordered-set) (filter % order))))ancestors (re-sort (:ancestors h) ts)descendants (re-sort (:descendants h) (reverse ts))](assoc h@@ -111,49 +152,41 @@:descendants descendants)))(defn derive- "Establishes a parent/child relationship between two keyword tags, similar to [[clojure.core/derive]].- Where it differs is that the order in which we derive keys is significant - it determines the order in which we return- its parents. Ancestors are returned according to a toposort, and descendants are returned in the opposite order."+ "Establishes a parent/child relationship between two keyword tags.++ Direct parents retain derivation order.+ Ancestors use leaves-to-roots topological order, and descendants use the reverse order."[h tag parent]- (assert (not= tag parent))- (assert (keyword tag))- (assert (keyword? parent))+ (assert (not= tag parent) "A tag cannot derive from itself.")+ (assert (keyword? tag) "Tag must be a keyword.")+ (assert (keyword? parent) "Parent must be a keyword.")(assert (::ordered? (meta h)) "This operation requires an ordered hierarchy.")- (let [tp (:parents h)- tc (:children h)]- (if (contains? (tp tag) parent)+ (let [parent-map (:parents h)+ child-map (:children h)]+ (if (contains? (parent-map tag) parent)h- (-> (assoc h :parents (update tp tag #(or % (ordered-set))))+ (-> (assoc h :parents (update parent-map tag #(or % (ordered-set))))(clojure.core/derive tag parent)- (assoc :children (update tc parent #(into (ordered-set tag) %)))+ (assoc :children (update child-map parent #(into (ordered-set tag) %)))calculate-derived-fields- (with-meta {::ordered? true})))))--(defn- first-common-tag- "Given two ordered sets corresponding to ancestors of two tags in a hierarchy, return the first ancestor of tag-a- which is also an ancestor of tag-b.- Returns nil if there is no such intersection."- [ancestors-a ancestors-b]- ;; I suspect this is not commutative, so this ordering is important - we care more about being close to tag-a.- (some ancestors-b ancestors-a))+ (with-meta (assoc (meta h) ::ordered? true))))))(defn first-common-ancestor- "Given two tags, return the first tag in the ancestral lineage of tag-a that's also in the ancestral lineage of tag-b.- Returns nil if there is no common ancestor.+ "Returns the first shared ancestor of `tag-a` and `tag-b` in the hierarchy's leaves-to-roots topological order.- NOTE: this is very similar to the notion of a Least Common Ancestor in a graph, but I'm not sure that its result- will always satisfy that property. We also want to emphasize the way it relates to the total ordering, hence- using different terminology."+ Each tag counts as its own ancestor.+ If either tag is nil, returns the other tag.+ Returns nil when there is no common ancestor."[h tag-a tag-b]- (let [ancestors-a (ancestors h tag-a)- ancestors-b (ancestors h tag-b)]- (cond- (nil? tag-a) tag-b- (nil? tag-b) tag-a- (= tag-a tag-b) tag-a- ;; This ordering is important - we want first ancestor of tag-a that is common, even if there are common- ;; ancestor which are closer to tag-b.- (contains? ancestors-b tag-a) tag-a- (contains? ancestors-a tag-b) tag-b- :else (first-common-tag ancestors-a ancestors-b))))+ (cond+ (nil? tag-a) tag-b+ (nil? tag-b) tag-a+ (= tag-a tag-b) tag-a+ ;; Tags with no ancestors, including tags absent from the hierarchy, come back as nil.+ :else (let [ancestors-a (or (ancestors h tag-a) #{})+ ancestors-b (or (ancestors h tag-b) #{})]+ (cond+ (contains? ancestors-b tag-a) tag-a+ (contains? ancestors-a tag-b) tag-b+ :else (some ancestors-b ancestors-a)))))diff --git a/test/metabase/util/ordered_hierarchy_test.clj b/test/metabase/util/ordered_hierarchy_test.cljindex 5372700b3f8..c82beb0d040 100644--- a/test/metabase/util/ordered_hierarchy_test.clj+++ b/test/metabase/util/ordered_hierarchy_test.clj@@ -18,7 +18,7 @@:right-angled-triangle:obtuse-triangle]))-(deftest make-hierarchy-test+(deftest ^:parallel make-hierarchy-test(testing "Hiccup structures have the expected topological order"(is (= [:isosceles-trapezoid:right-trapezoid@@ -56,3 +56,69 @@:right-angled-triangle [:triangle]:obtuse-triangle [:triangle]}(update-vals (:parents polygons) vec)))))++(deftest ^:parallel root-order-test+ (testing "A root without children is included"+ (is (= [:root]+ (vec (ordered-hierarchy/sorted-tags+ (ordered-hierarchy/make-hierarchy [:root]))))))+ (testing "Root order remains stable after the underlying map grows"+ (let [roots (mapv #(keyword (str "root-" %)) (range 12))+ hierarchy (apply ordered-hierarchy/make-hierarchy+ (map (fn [root]+ [root (keyword (str (name root) "-child"))])+ roots))]+ (is (= roots+ (filterv (set roots) (ordered-hierarchy/sorted-tags hierarchy)))))))++(deftest ^:parallel hierarchy-syntax-validation-test+ (testing "A basis must be a vector"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Hierarchy basis must be a vector"+ (ordered-hierarchy/make-hierarchy :root))))+ (testing "Vector tags must be keywords"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Hierarchy vectors must begin with a keyword tag"+ (ordered-hierarchy/make-hierarchy [])))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Hierarchy vectors must begin with a keyword tag"+ (ordered-hierarchy/make-hierarchy [:root []]))))+ (testing "Children must be keywords or vectors"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Hierarchy children must be keywords or vectors"+ (ordered-hierarchy/make-hierarchy [:root 1]))))+ (testing "Children may only be declared at a tag's first occurrence"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Children of :child may only be listed at its first occurrence"+ (ordered-hierarchy/make-hierarchy [:root :child [:child :grandchild]])))+ (testing "including when the later occurrence heads its own basis"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Children of :child may only be listed at its first occurrence"+ (ordered-hierarchy/make-hierarchy [:root :child] [:child :grandchild])))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Children of :root may only be listed at its first occurrence"+ (ordered-hierarchy/make-hierarchy [:root] [:root :child])))))+ (testing "Repeating a tag without listing children is allowed"+ (is (= [:child :root]+ (vec (ordered-hierarchy/sorted-tags+ (ordered-hierarchy/make-hierarchy [:root :child [:child]])))))+ (is (= [:child :root]+ (vec (ordered-hierarchy/sorted-tags+ (ordered-hierarchy/make-hierarchy [:root :child] [:child])))))))++(deftest ^:parallel derive-validation-test+ (is (thrown-with-msg? AssertionError+ #"Tag must be a keyword"+ (ordered-hierarchy/derive (ordered-hierarchy/make-hierarchy) 'child :parent))))++(deftest ^:parallel first-common-ancestor-test+ (testing "The shared ancestor closest to the first tag wins"+ (is (= :quadrilateral (ordered-hierarchy/first-common-ancestor polygons :square :trapezoid))))+ (testing "A tag counts as its own ancestor"+ (is (= :kite (ordered-hierarchy/first-common-ancestor polygons :square :kite))))+ (testing "Tags under separate roots have no common ancestor"+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :square :triangle)))+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :triangle :square))))+ (testing "Tags outside the hierarchy have no common ancestor"+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :square :circle)))+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :circle :square)))))