Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix ordered hierarchy ordering and validation (#80720)

Chris Trutera5bd647dc48d
diff --git a/src/metabase/util/ordered_hierarchy.clj b/src/metabase/util/ordered_hierarchy.clj
index ee1297352d2..026fecd0ce7 100644
--- a/src/metabase/util/ordered_hierarchy.clj
+++ b/src/metabase/util/ordered_hierarchy.clj
@@ -2,88 +2,125 @@
(ns metabase.util.ordered-hierarchy
(:refer-clojure :exclude [ancestors derive descendants make-hierarchy parents])
(:require
- [flatland.ordered.set :refer [ordered-set]]
- [medley.core :as m]))
+ [flatland.ordered.map :refer [ordered-map]]
+ [flatland.ordered.set :refer [ordered-set]]))
-(declare derive)
+(declare calculate-derived-fields derive)
+
+(defn- known-tag?
+ [h tag]
+ (or (contains? (:parents h) tag)
+ (contains? (:children h) tag)))
+
+(defn- check-first-occurrence!
+ [h [tag & children :as basis]]
+ (when (and (known-tag? h tag) (seq children))
+ (throw (ex-info (format "Children of %s may only be listed at its first occurrence" tag)
+ {:basis basis :tag tag}))))
+
+(defn- register-parent
+ [h parent]
+ (if (contains? (:children h) parent)
+ h
+ (assoc h :children (assoc (:children h) parent nil))))
(defn- derive-children
- [h [parent & children]]
- (reduce (fn [h child]
- (cond
- (keyword? child) (derive h child parent)
- (vector? child) (let [grandchild (first child)]
- (assert (not (and (contains? (:parents h) grandchild) (rest child)))
- (format "You may only list a %s's children at its first occurrence"
- grandchild))
- (derive-children (derive h grandchild parent)
- child))))
- h
- children))
+ [h [parent & children :as basis]]
+ (when-not (keyword? parent)
+ (throw (ex-info "Hierarchy vectors must begin with a keyword tag"
+ {:basis basis})))
+ (let [h (register-parent h parent)]
+ (if (seq children)
+ (reduce (fn [h child]
+ (cond
+ (keyword? child)
+ (derive h child parent)
+
+ (vector? child)
+ (let [grandchild (first child)]
+ (when-not (keyword? grandchild)
+ (throw (ex-info "Hierarchy vectors must begin with a keyword tag"
+ {:basis child})))
+ (check-first-occurrence! h child)
+ (derive-children (derive h grandchild parent) child))
+
+ :else
+ (throw (ex-info "Hierarchy children must be keywords or vectors"
+ {:basis basis :child child}))))
+ h
+ children)
+ (calculate-derived-fields h))))
(defn- derive-basis [h basis]
- (cond (vector? basis) (derive-children h basis)
- :else (throw (ex-info (str "Unsupported type for ordered-hierarchy: " (type basis))
- {:h h :basis basis}))))
+ (when-not (vector? basis)
+ (throw (ex-info (str "Hierarchy basis must be a vector, got " (type basis))
+ {:basis basis})))
+ (check-first-occurrence! h basis)
+ (derive-children h basis))
(defn make-hierarchy
- "Similar to [[clojure.core/make-hierarchy]], but the returned hierarchy has well-defined orderings for its sets.
+ "Creates a hierarchy whose sets have deterministic iteration order.
- Can take arguments to be treated as roots, defined using hiccup syntax.
+ Each optional basis uses Hiccup-style syntax: `[parent child [child grandchild ...] ...]`.
- !! WARNING !!
- Using [[clojure.core/derive]] with this will corrupt the ordering - you must use the implementation from this ns."
+ Always use [[derive]] to extend the result.
+ [[clojure.core/derive]] does not maintain this namespace's custom fields or metadata."
([]
(-> (clojure.core/make-hierarchy)
- (with-meta {::ordered? true})))
+ (assoc :children (ordered-map)
+ :sorted-tags (ordered-set))
+ (vary-meta assoc ::ordered? true)))
([& bases]
(reduce derive-basis (make-hierarchy) bases)))
(defn ancestors
- "Returns the immediate and indirect parents of tag, as established via derive. Earlier derivations are shown first.
- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."
+ "Returns the immediate and indirect parents of `tag` in leaves-to-roots topological order.
+
+ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."
[h tag]
(clojure.core/ancestors h tag))
(defn descendants
- "Returns the immediate and indirect children of tag, as established via derive. Earlier derivations are shown later.
- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."
+ "Returns the immediate and indirect children of `tag` in reverse topological order.
+
+ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."
[h tag]
(clojure.core/descendants h tag))
(defn parents
- "Returns the immediate parents of tag, as established via derive. Earlier derivations are shown first.
- This method is just a proxy, it exists only to prevent accidentally using the global hierarchy."
+ "Returns the immediate parents of `tag` in derivation order.
+
+ This function deliberately requires a hierarchy argument, preventing accidental use of Clojure's global hierarchy."
[h tag]
(clojure.core/parents h tag))
(defn children
- "Returns the immediate children of tag, as established via derive. Later derivations are shown first."
+ "Returns the immediate children of `tag` in reverse derivation order."
[h tag]
- (-> h :children tag))
+ (get-in h [:children tag]))
-(defn- toposort-visit [g state n]
+(defn- toposort-visit [graph state node]
(let [{:keys [processing processed]} state]
- (when (contains? processing n)
+ (when (contains? processing node)
(throw (ex-info "Cycle in graph" {:cause ::cyclic-graph
- :cycle-node n
+ :cycle-node node
:state state
- :graph g})))
- (if (contains? processed n)
+ :graph graph})))
+ (if (contains? processed node)
state
- (let [children (get g n)
- init-state (update state :processing conj n)
- post-state (reduce (partial toposort-visit g)
+ (let [children (get graph node)
+ init-state (update state :processing conj node)
+ post-state (reduce (partial toposort-visit graph)
init-state
- ;; Iterate in reverse as that gives us oldest to newest
+ ;; Children are stored newest-first; visit them in derivation order.
(reverse children))]
(-> post-state
- (update :processed conj n)
- (update :processing disj n))))))
+ (update :processed conj node)
+ (update :processing disj node))))))
(defn- toposort
- "Use Trajan's DPS method to return an ordered set of nodes where every node is preceded by all its children, and
- those children are likewise enumerated in the order they were registered as children of that node."
+ "Performs a depth-first topological sort.
+ Children precede their parents, and sibling ties follow derivation order."
[roots graph]
(->> roots
(reduce (partial toposort-visit graph)
@@ -91,18 +128,22 @@
:processed (ordered-set)})
:processed))
-(defn sorted-tags
- "An ordered set of all tags within the hierarchy, topologically sorted from the leaves up, and following the edges
- according to the order they were added in."
+(defn- calculate-sorted-tags
[h]
(let [roots (reduce disj
(into (ordered-set) (keys (:children h)))
(keys (:parents h)))]
(toposort roots (:children h))))
+(defn sorted-tags
+ "Returns all tags in leaves-to-roots topological order, using derivation order to break ties."
+ [h]
+ (or (:sorted-tags h)
+ (calculate-sorted-tags h)))
+
(defn- calculate-derived-fields [h]
- (let [ts (sorted-tags h)
- re-sort (fn [m sorted] (m/map-vals #(into (ordered-set) (filter % sorted)) m))
+ (let [ts (calculate-sorted-tags h)
+ re-sort (fn [m order] (update-vals m #(into (ordered-set) (filter % order))))
ancestors (re-sort (:ancestors h) ts)
descendants (re-sort (:descendants h) (reverse ts))]
(assoc h
@@ -111,49 +152,41 @@
:descendants descendants)))
(defn derive
- "Establishes a parent/child relationship between two keyword tags, similar to [[clojure.core/derive]].
- Where it differs is that the order in which we derive keys is significant - it determines the order in which we return
- its parents. Ancestors are returned according to a toposort, and descendants are returned in the opposite order."
+ "Establishes a parent/child relationship between two keyword tags.
+
+ Direct parents retain derivation order.
+ Ancestors use leaves-to-roots topological order, and descendants use the reverse order."
[h tag parent]
- (assert (not= tag parent))
- (assert (keyword tag))
- (assert (keyword? parent))
+ (assert (not= tag parent) "A tag cannot derive from itself.")
+ (assert (keyword? tag) "Tag must be a keyword.")
+ (assert (keyword? parent) "Parent must be a keyword.")
(assert (::ordered? (meta h)) "This operation requires an ordered hierarchy.")
- (let [tp (:parents h)
- tc (:children h)]
- (if (contains? (tp tag) parent)
+ (let [parent-map (:parents h)
+ child-map (:children h)]
+ (if (contains? (parent-map tag) parent)
h
- (-> (assoc h :parents (update tp tag #(or % (ordered-set))))
+ (-> (assoc h :parents (update parent-map tag #(or % (ordered-set))))
(clojure.core/derive tag parent)
- (assoc :children (update tc parent #(into (ordered-set tag) %)))
+ (assoc :children (update child-map parent #(into (ordered-set tag) %)))
calculate-derived-fields
- (with-meta {::ordered? true})))))
-
-(defn- first-common-tag
- "Given two ordered sets corresponding to ancestors of two tags in a hierarchy, return the first ancestor of tag-a
- which is also an ancestor of tag-b.
- Returns nil if there is no such intersection."
- [ancestors-a ancestors-b]
- ;; I suspect this is not commutative, so this ordering is important - we care more about being close to tag-a.
- (some ancestors-b ancestors-a))
+ (with-meta (assoc (meta h) ::ordered? true))))))
(defn first-common-ancestor
- "Given two tags, return the first tag in the ancestral lineage of tag-a that's also in the ancestral lineage of tag-b.
- Returns nil if there is no common ancestor.
+ "Returns the first shared ancestor of `tag-a` and `tag-b` in the hierarchy's leaves-to-roots topological order.
- NOTE: this is very similar to the notion of a Least Common Ancestor in a graph, but I'm not sure that its result
- will always satisfy that property. We also want to emphasize the way it relates to the total ordering, hence
- using different terminology."
+ Each tag counts as its own ancestor.
+ If either tag is nil, returns the other tag.
+ Returns nil when there is no common ancestor."
[h tag-a tag-b]
- (let [ancestors-a (ancestors h tag-a)
- ancestors-b (ancestors h tag-b)]
- (cond
- (nil? tag-a) tag-b
- (nil? tag-b) tag-a
- (= tag-a tag-b) tag-a
- ;; This ordering is important - we want first ancestor of tag-a that is common, even if there are common
- ;; ancestor which are closer to tag-b.
- (contains? ancestors-b tag-a) tag-a
- (contains? ancestors-a tag-b) tag-b
- :else (first-common-tag ancestors-a ancestors-b))))
+ (cond
+ (nil? tag-a) tag-b
+ (nil? tag-b) tag-a
+ (= tag-a tag-b) tag-a
+ ;; Tags with no ancestors, including tags absent from the hierarchy, come back as nil.
+ :else (let [ancestors-a (or (ancestors h tag-a) #{})
+ ancestors-b (or (ancestors h tag-b) #{})]
+ (cond
+ (contains? ancestors-b tag-a) tag-a
+ (contains? ancestors-a tag-b) tag-b
+ :else (some ancestors-b ancestors-a)))))
diff --git a/test/metabase/util/ordered_hierarchy_test.clj b/test/metabase/util/ordered_hierarchy_test.clj
index 5372700b3f8..c82beb0d040 100644
--- a/test/metabase/util/ordered_hierarchy_test.clj
+++ b/test/metabase/util/ordered_hierarchy_test.clj
@@ -18,7 +18,7 @@
:right-angled-triangle
:obtuse-triangle]))
-(deftest make-hierarchy-test
+(deftest ^:parallel make-hierarchy-test
(testing "Hiccup structures have the expected topological order"
(is (= [:isosceles-trapezoid
:right-trapezoid
@@ -56,3 +56,69 @@
:right-angled-triangle [:triangle]
:obtuse-triangle [:triangle]}
(update-vals (:parents polygons) vec)))))
+
+(deftest ^:parallel root-order-test
+ (testing "A root without children is included"
+ (is (= [:root]
+ (vec (ordered-hierarchy/sorted-tags
+ (ordered-hierarchy/make-hierarchy [:root]))))))
+ (testing "Root order remains stable after the underlying map grows"
+ (let [roots (mapv #(keyword (str "root-" %)) (range 12))
+ hierarchy (apply ordered-hierarchy/make-hierarchy
+ (map (fn [root]
+ [root (keyword (str (name root) "-child"))])
+ roots))]
+ (is (= roots
+ (filterv (set roots) (ordered-hierarchy/sorted-tags hierarchy)))))))
+
+(deftest ^:parallel hierarchy-syntax-validation-test
+ (testing "A basis must be a vector"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Hierarchy basis must be a vector"
+ (ordered-hierarchy/make-hierarchy :root))))
+ (testing "Vector tags must be keywords"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Hierarchy vectors must begin with a keyword tag"
+ (ordered-hierarchy/make-hierarchy [])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Hierarchy vectors must begin with a keyword tag"
+ (ordered-hierarchy/make-hierarchy [:root []]))))
+ (testing "Children must be keywords or vectors"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Hierarchy children must be keywords or vectors"
+ (ordered-hierarchy/make-hierarchy [:root 1]))))
+ (testing "Children may only be declared at a tag's first occurrence"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Children of :child may only be listed at its first occurrence"
+ (ordered-hierarchy/make-hierarchy [:root :child [:child :grandchild]])))
+ (testing "including when the later occurrence heads its own basis"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Children of :child may only be listed at its first occurrence"
+ (ordered-hierarchy/make-hierarchy [:root :child] [:child :grandchild])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Children of :root may only be listed at its first occurrence"
+ (ordered-hierarchy/make-hierarchy [:root] [:root :child])))))
+ (testing "Repeating a tag without listing children is allowed"
+ (is (= [:child :root]
+ (vec (ordered-hierarchy/sorted-tags
+ (ordered-hierarchy/make-hierarchy [:root :child [:child]])))))
+ (is (= [:child :root]
+ (vec (ordered-hierarchy/sorted-tags
+ (ordered-hierarchy/make-hierarchy [:root :child] [:child])))))))
+
+(deftest ^:parallel derive-validation-test
+ (is (thrown-with-msg? AssertionError
+ #"Tag must be a keyword"
+ (ordered-hierarchy/derive (ordered-hierarchy/make-hierarchy) 'child :parent))))
+
+(deftest ^:parallel first-common-ancestor-test
+ (testing "The shared ancestor closest to the first tag wins"
+ (is (= :quadrilateral (ordered-hierarchy/first-common-ancestor polygons :square :trapezoid))))
+ (testing "A tag counts as its own ancestor"
+ (is (= :kite (ordered-hierarchy/first-common-ancestor polygons :square :kite))))
+ (testing "Tags under separate roots have no common ancestor"
+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :square :triangle)))
+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :triangle :square))))
+ (testing "Tags outside the hierarchy have no common ancestor"
+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :square :circle)))
+ (is (nil? (ordered-hierarchy/first-common-ancestor polygons :circle :square)))))