Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove :metabase.lib.field/original-temporal-unit and document friends (#69627)

metamben04e14dcee2e9
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljc
index e86ec788872..39f4ae32ea7 100644
--- a/src/metabase/lib/convert.cljc
+++ b/src/metabase/lib/convert.cljc
@@ -22,7 +22,7 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.performance :as perf :refer [mapv some select-keys not-empty #?(:clj doseq) #?(:clj for)]])
+ [metabase.util.performance :refer [mapv some select-keys not-empty #?(:clj doseq) #?(:clj for)]])
#?@(:cljs [(:require-macros [metabase.lib.convert :refer [with-aggregation-list]])]))
(def ^:private ^:dynamic *pMBQL-uuid->legacy-index*
@@ -487,10 +487,6 @@
([m]
(into {} (disqualify) m)))
-(def ^:private options-preserved-in-legacy
- "Map of option keys in pMBQL to their legacy names. Keys are renamed before [[disqualify]] drops all namespaced keys."
- {:metabase.lib.field/original-temporal-unit :original-temporal-unit})
-
(mu/defn- options->legacy-MBQL :- [:maybe [:map {:min 1}]]
"Convert an options map in an MBQL clause to the equivalent shape for legacy MBQL. Remove `:lib/*` keys and
`:effective-type`, which is not used in options maps in legacy MBQL."
@@ -499,10 +495,7 @@
;; Following construct ensures that transformation mbql -> pmbql -> mbql, does not add base-type where those
;; were not present originally. Base types are added in [[metabase.lib.query/add-types-to-fields]].
(:metabase.lib.query/transformation-added-base-type m)
- (dissoc :metabase.lib.query/transformation-added-base-type :base-type)
-
- ;; Removing the namespaces from a few
- true (perf/update-keys #(get options-preserved-in-legacy % %)))
+ (dissoc :metabase.lib.query/transformation-added-base-type :base-type))
(into {} (comp (disqualify)
;; remove `:effective-type` if `:base-type` is not present OR if it's the same as `:base-type`.
(remove (let [keys-to-remove (if (or (nil? (:base-type m))
diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljc
index e4ed99bca70..ecfe3c13adb 100644
--- a/src/metabase/lib/field.cljc
+++ b/src/metabase/lib/field.cljc
@@ -296,16 +296,13 @@
(defmethod lib.temporal-bucket/with-temporal-bucket-method :metadata/column
[metadata unit]
- (let [original-effective-type ((some-fn ::original-effective-type :effective-type :base-type) metadata)
- original-temporal-unit ((some-fn ::original-temporal-unit ::temporal-unit) metadata)]
+ (let [original-effective-type ((some-fn ::original-effective-type :effective-type :base-type) metadata)]
(if unit
(-> metadata
(assoc ::temporal-unit unit)
- (m/assoc-some ::original-effective-type original-effective-type
- ::original-temporal-unit original-temporal-unit))
+ (m/assoc-some ::original-effective-type original-effective-type))
(cond-> (dissoc metadata ::temporal-unit ::original-effective-type)
- original-effective-type (assoc :effective-type original-effective-type)
- original-temporal-unit (assoc ::original-temporal-unit original-temporal-unit)))))
+ original-effective-type (assoc :effective-type original-effective-type)))))
(defmethod lib.temporal-bucket/available-temporal-buckets-method :field
[query stage-number field-ref]
@@ -403,8 +400,7 @@
[:base-type
:inherited-temporal-unit
:lib/original-binning
- ::original-effective-type
- ::original-temporal-unit])
+ ::original-effective-type])
{:metabase.lib.field/binning :binning
:metabase.lib.field/temporal-unit :temporal-unit
:lib/ref-name :name
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index bd91cc0e072..fb0150654c1 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -136,8 +136,7 @@
:effective-type
:display-name
:metabase.lib.query/transformation-added-base-type
- :metabase.lib.field/original-effective-type
- :metabase.lib.field/original-temporal-unit})
+ :metabase.lib.field/original-effective-type})
(def ^:private opts-propagated-renamed-keys
"Keys in `:field` opts that get copied into column metadata with different keys when they have non-nil values.
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index a3c9537a34e..7cf11b4ba2a 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -385,8 +385,9 @@
[:metabase.lib.field/temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]
[:metabase.lib.field/binning {:optional true} [:maybe ::lib.schema.binning/binning]]
;;
- ;; if temporal bucketing or binning happened in the previous stage, respectively, they should get propagated as
- ;; these keys.
+ ;; If temporal bucketing or binning happened in a previous stage, they are propagated as the keys below.
+ ;; `:inherited-temporal-unit` signals that this column was already bucketed upstream, so the default temporal
+ ;; unit becomes `:inherited` rather than a type-based default like `:month`, preventing double-bucketing.
[:inherited-temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]
[:lib/original-binning {:optional true} [:maybe ::lib.schema.binning/binning]]
;; name of the expression where this column metadata came from. Should only be included for expressions introduced
diff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljc
index 54a80cfeb0b..86b12826e5b 100644
--- a/src/metabase/lib/schema/ref.cljc
+++ b/src/metabase/lib/schema/ref.cljc
@@ -80,7 +80,6 @@
[:binning {:optional true} [:ref ::binning/binning]]
[:lib/original-binning {:optional true} [:ref ::binning/binning]]
[:metabase.lib.field/original-effective-type {:optional true} [:ref ::common/base-type]]
- [:metabase.lib.field/original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]
;;
;; For implicitly joinable columns, the ID of the FK field used to perform the implicit join.
;; E.g. if the query is against `ORDERS` and the field ref is for `PRODUCTS.CATEGORY`, then `:source-field`
@@ -127,13 +126,17 @@
;; identifies an implicit join.
[:source-field-join-alias {:optional true} ::common/non-blank-string]
;;
- ;; Inherited temporal unit captures the temporal unit, that has been set on a ref, for next stages. It is attached
- ;; _to a column_, which is created from this ref by means of `returned-columns`, ie. is visible [inherited temporal
- ;; unit] in next stages only. This information is used eg. to help pick a default _temporal unit_ for columns that
- ;; are bucketed -- if a column contains `:inherited-temporal-unit`, it was bucketed already in previous stages,
- ;; so nil default picked to avoid another round of bucketing. Shall user bucket the column again, they have to
- ;; select the bucketing explicitly in QB.
- [:inherited-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]]]
+ ;; Records the temporal unit applied to this field in a previous stage. Propagated from `:temporal-unit` onto
+ ;; column metadata during `returned-columns`, so it is only visible in subsequent stages. Used to pick the
+ ;; default temporal unit for already-bucketed columns: when present, the default becomes `:inherited` instead
+ ;; of a type-based unit like `:month`, preventing accidental double-bucketing in the UI.
+ [:inherited-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]
+ ;;
+ ;; Legacy key. Records the temporal unit that was originally on a field ref before it was changed or removed.
+ ;; Produced by older queries and the `reconcile-breakout-and-order-by-bucketing` QP middleware. Used as a
+ ;; fallback in `nest-breakouts` to determine column granularity when `:temporal-unit` is nil or `:default`.
+ ;; Not produced by MLv2 code; new queries will not contain this key.
+ [:original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]]]
(common/disallowed-keys
{:strategy ":binning keys like :strategy are not allowed at the top level of :field options."})
;; If `:base-type` is specified, the `:temporal-unit` must make sense, e.g. no bucketing by `:year`for a
diff --git a/src/metabase/lib/temporal_bucket.cljc b/src/metabase/lib/temporal_bucket.cljc
index 925733a1047..ef7e252355c 100644
--- a/src/metabase/lib/temporal_bucket.cljc
+++ b/src/metabase/lib/temporal_bucket.cljc
@@ -461,8 +461,7 @@
;; we want to remove it later. We will record this with the key `::original-effective-type`. Note that changing the
;; unit multiple times should keep the original first value of `::original-effective-type`.
(if unit
- (let [original-temporal-unit ((some-fn :metabase.lib.field/original-temporal-unit :temporal-unit) options)
- extraction-unit? (contains? lib.schema.temporal-bucketing/datetime-extraction-units unit)
+ (let [extraction-unit? (contains? lib.schema.temporal-bucketing/datetime-extraction-units unit)
original-effective-type ((some-fn :metabase.lib.field/original-effective-type :effective-type :base-type)
options)
new-effective-type (if extraction-unit?
@@ -471,18 +470,14 @@
options (-> options
(assoc :temporal-unit unit
:effective-type new-effective-type)
- (m/assoc-some :metabase.lib.field/original-effective-type original-effective-type
- :metabase.lib.field/original-temporal-unit original-temporal-unit))]
+ (m/assoc-some :metabase.lib.field/original-effective-type original-effective-type))]
[tag options id-or-name])
- ;; `unit` is `nil`: remove the temporal bucket and remember it :metabase.lib.field/original-temporal-unit.
+ ;; `unit` is `nil`: remove the temporal bucket.
(let [original-effective-type (:metabase.lib.field/original-effective-type options)
- original-temporal-unit ((some-fn :metabase.lib.field/original-temporal-unit :temporal-unit) options)
options (cond-> (dissoc options :temporal-unit)
original-effective-type
(-> (assoc :effective-type original-effective-type)
- (dissoc :metabase.lib.field/original-effective-type))
- original-temporal-unit
- (assoc :metabase.lib.field/original-temporal-unit original-temporal-unit))]
+ (dissoc :metabase.lib.field/original-effective-type)))]
[tag options id-or-name])))
(defn- ends-with-temporal-unit?
diff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.clj
index 0ad7de7ff35..a8ff12593e2 100644
--- a/src/metabase/query_processor/util/nest_query.clj
+++ b/src/metabase/query_processor/util/nest_query.clj
@@ -57,7 +57,6 @@
;; allowed if the binning/bucketing is happening at this stage but they're no
;; longer happening here so leaving them in place would be incorrect.
(lib/update-options dissoc
- :metabase.lib.field/original-temporal-unit
:original-temporal-unit
:lib/original-binning))]
(vswap! refs conj! unbucketed-ref)))))))
diff --git a/src/metabase/query_processor/util/transformations/nest_breakouts.clj b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
index 95709f6c7dc..20aa373c50c 100644
--- a/src/metabase/query_processor/util/transformations/nest_breakouts.clj
+++ b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
@@ -96,7 +96,6 @@
temporal-unit
(or (:original-temporal-unit temporal-attributes)
(:inherited-temporal-unit temporal-attributes)
- (:metabase.lib.field/original-temporal-unit temporal-attributes)
temporal-unit))))
(defn- column-granularity
diff --git a/test/metabase/lib/field_test.cljc b/test/metabase/lib/field_test.cljc
index 355b570ef92..54e64533486 100644
--- a/test/metabase/lib/field_test.cljc
+++ b/test/metabase/lib/field_test.cljc
@@ -54,7 +54,6 @@
:lib/source-column-alias "CREATED_AT"
:lib/type :metadata/column
:metabase.lib.field/original-effective-type :type/DateTime
- :metabase.lib.field/original-temporal-unit :year
:metabase.lib.field/temporal-unit :year}))))
(defn grandparent-parent-child-id [field]
@@ -247,8 +246,8 @@
(doseq [[unit effective-type] {:month :type/Date
:month-of-year :type/Integer}
:let [field-metadata (get-in temporal-bucketing-mock-metadata [:fields :date])]
- [what x update-props] [["column metadata" field-metadata (fn [x f & args] (apply f x args))]
- ["field ref" (lib/ref field-metadata) lib.options/update-options]]
+ [what x] [["column metadata" field-metadata]
+ ["field ref" (lib/ref field-metadata)]]
:let [x' (lib/with-temporal-bucket x unit)]]
(testing (str what " unit = " unit "\n\n" (u/pprint-to-str x') "\n")
(testing "should calculate correct effective type"
@@ -269,14 +268,12 @@
(testing "remove the temporal unit"
(let [x'' (lib/with-temporal-bucket x' nil)]
(is (nil? (lib/temporal-bucket x'')))
- (is (= x
- (update-props x'' dissoc :metabase.lib.field/original-temporal-unit)))))
+ (is (= x x''))))
(testing "change the temporal unit, THEN remove it"
(let [x'' (lib/with-temporal-bucket x' :quarter-of-year)
x''' (lib/with-temporal-bucket x'' nil)]
(is (nil? (lib/temporal-bucket x''')))
- (is (= x
- (update-props x''' dissoc :metabase.lib.field/original-temporal-unit))))))))
+ (is (= x x''')))))))
(deftest ^:parallel available-temporal-buckets-test
(doseq [{:keys [metadata expected-options selected-index selected-unit]}
diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljc
index 59dcc71aa4e..8dc139fd147 100644
--- a/test/metabase/lib/remove_replace_test.cljc
+++ b/test/metabase/lib/remove_replace_test.cljc
@@ -764,16 +764,16 @@
breakout-col (->> (lib/breakoutable-columns query)
(m/find-first (comp #{"LAST_LOGIN"} :name)))
month (lib/with-temporal-bucket breakout-col :month)
- day (assoc (lib/with-temporal-bucket breakout-col :day)
- :metabase.lib.field/original-temporal-unit :month)
+ day (lib/with-temporal-bucket breakout-col :day)
q2 (-> query
(lib/breakout month))
cols (lib/orderable-columns q2)
q3 (-> q2
(lib/order-by (first cols))
(lib/replace-clause (first (lib/breakouts q2)) day))]
- (is (= (get-in q3 [:stages 0 :breakout 0 1 :metabase.lib.field/original-temporal-unit])
- (get-in q3 [:stages 0 :order-by 0 2 1 :metabase.lib.field/original-temporal-unit])))))))
+ (is (= (get-in q3 [:stages 0 :breakout 0 1 :temporal-unit])
+ (get-in q3 [:stages 0 :order-by 0 2 1 :temporal-unit])
+ :day))))))
(deftest ^:parallel empty-first-stage-gets-merged-test
(testing "issue #45041"