The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove :metabase.lib.field/original-temporal-unit and document friends (#69627)
metamben04e14dcee2e9
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljcindex e86ec788872..39f4ae32ea7 100644--- a/src/metabase/lib/convert.cljc+++ b/src/metabase/lib/convert.cljc@@ -22,7 +22,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :as perf :refer [mapv some select-keys not-empty #?(:clj doseq) #?(:clj for)]])+ [metabase.util.performance :refer [mapv some select-keys not-empty #?(:clj doseq) #?(:clj for)]])#?@(:cljs [(:require-macros [metabase.lib.convert :refer [with-aggregation-list]])]))(def ^:private ^:dynamic *pMBQL-uuid->legacy-index*@@ -487,10 +487,6 @@([m](into {} (disqualify) m)))-(def ^:private options-preserved-in-legacy- "Map of option keys in pMBQL to their legacy names. Keys are renamed before [[disqualify]] drops all namespaced keys."- {:metabase.lib.field/original-temporal-unit :original-temporal-unit})-(mu/defn- options->legacy-MBQL :- [:maybe [:map {:min 1}]]"Convert an options map in an MBQL clause to the equivalent shape for legacy MBQL. Remove `:lib/*` keys and`:effective-type`, which is not used in options maps in legacy MBQL."@@ -499,10 +495,7 @@;; Following construct ensures that transformation mbql -> pmbql -> mbql, does not add base-type where those;; were not present originally. Base types are added in [[metabase.lib.query/add-types-to-fields]].(:metabase.lib.query/transformation-added-base-type m)- (dissoc :metabase.lib.query/transformation-added-base-type :base-type)-- ;; Removing the namespaces from a few- true (perf/update-keys #(get options-preserved-in-legacy % %)))+ (dissoc :metabase.lib.query/transformation-added-base-type :base-type))(into {} (comp (disqualify);; remove `:effective-type` if `:base-type` is not present OR if it's the same as `:base-type`.(remove (let [keys-to-remove (if (or (nil? (:base-type m))diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex e4ed99bca70..ecfe3c13adb 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -296,16 +296,13 @@(defmethod lib.temporal-bucket/with-temporal-bucket-method :metadata/column[metadata unit]- (let [original-effective-type ((some-fn ::original-effective-type :effective-type :base-type) metadata)- original-temporal-unit ((some-fn ::original-temporal-unit ::temporal-unit) metadata)]+ (let [original-effective-type ((some-fn ::original-effective-type :effective-type :base-type) metadata)](if unit(-> metadata(assoc ::temporal-unit unit)- (m/assoc-some ::original-effective-type original-effective-type- ::original-temporal-unit original-temporal-unit))+ (m/assoc-some ::original-effective-type original-effective-type))(cond-> (dissoc metadata ::temporal-unit ::original-effective-type)- original-effective-type (assoc :effective-type original-effective-type)- original-temporal-unit (assoc ::original-temporal-unit original-temporal-unit)))))+ original-effective-type (assoc :effective-type original-effective-type)))))(defmethod lib.temporal-bucket/available-temporal-buckets-method :field[query stage-number field-ref]@@ -403,8 +400,7 @@[:base-type:inherited-temporal-unit:lib/original-binning- ::original-effective-type- ::original-temporal-unit])+ ::original-effective-type]){:metabase.lib.field/binning :binning:metabase.lib.field/temporal-unit :temporal-unit:lib/ref-name :namediff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex bd91cc0e072..fb0150654c1 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -136,8 +136,7 @@:effective-type:display-name:metabase.lib.query/transformation-added-base-type- :metabase.lib.field/original-effective-type- :metabase.lib.field/original-temporal-unit})+ :metabase.lib.field/original-effective-type})(def ^:private opts-propagated-renamed-keys"Keys in `:field` opts that get copied into column metadata with different keys when they have non-nil values.diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex a3c9537a34e..7cf11b4ba2a 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -385,8 +385,9 @@[:metabase.lib.field/temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]][:metabase.lib.field/binning {:optional true} [:maybe ::lib.schema.binning/binning]];;- ;; if temporal bucketing or binning happened in the previous stage, respectively, they should get propagated as- ;; these keys.+ ;; If temporal bucketing or binning happened in a previous stage, they are propagated as the keys below.+ ;; `:inherited-temporal-unit` signals that this column was already bucketed upstream, so the default temporal+ ;; unit becomes `:inherited` rather than a type-based default like `:month`, preventing double-bucketing.[:inherited-temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]][:lib/original-binning {:optional true} [:maybe ::lib.schema.binning/binning]];; name of the expression where this column metadata came from. Should only be included for expressions introduceddiff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljcindex 54a80cfeb0b..86b12826e5b 100644--- a/src/metabase/lib/schema/ref.cljc+++ b/src/metabase/lib/schema/ref.cljc@@ -80,7 +80,6 @@[:binning {:optional true} [:ref ::binning/binning]][:lib/original-binning {:optional true} [:ref ::binning/binning]][:metabase.lib.field/original-effective-type {:optional true} [:ref ::common/base-type]]- [:metabase.lib.field/original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]];;;; For implicitly joinable columns, the ID of the FK field used to perform the implicit join.;; E.g. if the query is against `ORDERS` and the field ref is for `PRODUCTS.CATEGORY`, then `:source-field`@@ -127,13 +126,17 @@;; identifies an implicit join.[:source-field-join-alias {:optional true} ::common/non-blank-string];;- ;; Inherited temporal unit captures the temporal unit, that has been set on a ref, for next stages. It is attached- ;; _to a column_, which is created from this ref by means of `returned-columns`, ie. is visible [inherited temporal- ;; unit] in next stages only. This information is used eg. to help pick a default _temporal unit_ for columns that- ;; are bucketed -- if a column contains `:inherited-temporal-unit`, it was bucketed already in previous stages,- ;; so nil default picked to avoid another round of bucketing. Shall user bucket the column again, they have to- ;; select the bucketing explicitly in QB.- [:inherited-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]]]+ ;; Records the temporal unit applied to this field in a previous stage. Propagated from `:temporal-unit` onto+ ;; column metadata during `returned-columns`, so it is only visible in subsequent stages. Used to pick the+ ;; default temporal unit for already-bucketed columns: when present, the default becomes `:inherited` instead+ ;; of a type-based unit like `:month`, preventing accidental double-bucketing in the UI.+ [:inherited-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]+ ;;+ ;; Legacy key. Records the temporal unit that was originally on a field ref before it was changed or removed.+ ;; Produced by older queries and the `reconcile-breakout-and-order-by-bucketing` QP middleware. Used as a+ ;; fallback in `nest-breakouts` to determine column granularity when `:temporal-unit` is nil or `:default`.+ ;; Not produced by MLv2 code; new queries will not contain this key.+ [:original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]]](common/disallowed-keys{:strategy ":binning keys like :strategy are not allowed at the top level of :field options."});; If `:base-type` is specified, the `:temporal-unit` must make sense, e.g. no bucketing by `:year`for adiff --git a/src/metabase/lib/temporal_bucket.cljc b/src/metabase/lib/temporal_bucket.cljcindex 925733a1047..ef7e252355c 100644--- a/src/metabase/lib/temporal_bucket.cljc+++ b/src/metabase/lib/temporal_bucket.cljc@@ -461,8 +461,7 @@;; we want to remove it later. We will record this with the key `::original-effective-type`. Note that changing the;; unit multiple times should keep the original first value of `::original-effective-type`.(if unit- (let [original-temporal-unit ((some-fn :metabase.lib.field/original-temporal-unit :temporal-unit) options)- extraction-unit? (contains? lib.schema.temporal-bucketing/datetime-extraction-units unit)+ (let [extraction-unit? (contains? lib.schema.temporal-bucketing/datetime-extraction-units unit)original-effective-type ((some-fn :metabase.lib.field/original-effective-type :effective-type :base-type)options)new-effective-type (if extraction-unit?@@ -471,18 +470,14 @@options (-> options(assoc :temporal-unit unit:effective-type new-effective-type)- (m/assoc-some :metabase.lib.field/original-effective-type original-effective-type- :metabase.lib.field/original-temporal-unit original-temporal-unit))]+ (m/assoc-some :metabase.lib.field/original-effective-type original-effective-type))][tag options id-or-name])- ;; `unit` is `nil`: remove the temporal bucket and remember it :metabase.lib.field/original-temporal-unit.+ ;; `unit` is `nil`: remove the temporal bucket.(let [original-effective-type (:metabase.lib.field/original-effective-type options)- original-temporal-unit ((some-fn :metabase.lib.field/original-temporal-unit :temporal-unit) options)options (cond-> (dissoc options :temporal-unit)original-effective-type(-> (assoc :effective-type original-effective-type)- (dissoc :metabase.lib.field/original-effective-type))- original-temporal-unit- (assoc :metabase.lib.field/original-temporal-unit original-temporal-unit))]+ (dissoc :metabase.lib.field/original-effective-type)))][tag options id-or-name])))(defn- ends-with-temporal-unit?diff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.cljindex 0ad7de7ff35..a8ff12593e2 100644--- a/src/metabase/query_processor/util/nest_query.clj+++ b/src/metabase/query_processor/util/nest_query.clj@@ -57,7 +57,6 @@;; allowed if the binning/bucketing is happening at this stage but they're no;; longer happening here so leaving them in place would be incorrect.(lib/update-options dissoc- :metabase.lib.field/original-temporal-unit:original-temporal-unit:lib/original-binning))](vswap! refs conj! unbucketed-ref)))))))diff --git a/src/metabase/query_processor/util/transformations/nest_breakouts.clj b/src/metabase/query_processor/util/transformations/nest_breakouts.cljindex 95709f6c7dc..20aa373c50c 100644--- a/src/metabase/query_processor/util/transformations/nest_breakouts.clj+++ b/src/metabase/query_processor/util/transformations/nest_breakouts.clj@@ -96,7 +96,6 @@temporal-unit(or (:original-temporal-unit temporal-attributes)(:inherited-temporal-unit temporal-attributes)- (:metabase.lib.field/original-temporal-unit temporal-attributes)temporal-unit))))(defn- column-granularitydiff --git a/test/metabase/lib/field_test.cljc b/test/metabase/lib/field_test.cljcindex 355b570ef92..54e64533486 100644--- a/test/metabase/lib/field_test.cljc+++ b/test/metabase/lib/field_test.cljc@@ -54,7 +54,6 @@:lib/source-column-alias "CREATED_AT":lib/type :metadata/column:metabase.lib.field/original-effective-type :type/DateTime- :metabase.lib.field/original-temporal-unit :year:metabase.lib.field/temporal-unit :year}))))(defn grandparent-parent-child-id [field]@@ -247,8 +246,8 @@(doseq [[unit effective-type] {:month :type/Date:month-of-year :type/Integer}:let [field-metadata (get-in temporal-bucketing-mock-metadata [:fields :date])]- [what x update-props] [["column metadata" field-metadata (fn [x f & args] (apply f x args))]- ["field ref" (lib/ref field-metadata) lib.options/update-options]]+ [what x] [["column metadata" field-metadata]+ ["field ref" (lib/ref field-metadata)]]:let [x' (lib/with-temporal-bucket x unit)]](testing (str what " unit = " unit "\n\n" (u/pprint-to-str x') "\n")(testing "should calculate correct effective type"@@ -269,14 +268,12 @@(testing "remove the temporal unit"(let [x'' (lib/with-temporal-bucket x' nil)](is (nil? (lib/temporal-bucket x'')))- (is (= x- (update-props x'' dissoc :metabase.lib.field/original-temporal-unit)))))+ (is (= x x''))))(testing "change the temporal unit, THEN remove it"(let [x'' (lib/with-temporal-bucket x' :quarter-of-year)x''' (lib/with-temporal-bucket x'' nil)](is (nil? (lib/temporal-bucket x''')))- (is (= x- (update-props x''' dissoc :metabase.lib.field/original-temporal-unit))))))))+ (is (= x x''')))))))(deftest ^:parallel available-temporal-buckets-test(doseq [{:keys [metadata expected-options selected-index selected-unit]}diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljcindex 59dcc71aa4e..8dc139fd147 100644--- a/test/metabase/lib/remove_replace_test.cljc+++ b/test/metabase/lib/remove_replace_test.cljc@@ -764,16 +764,16 @@breakout-col (->> (lib/breakoutable-columns query)(m/find-first (comp #{"LAST_LOGIN"} :name)))month (lib/with-temporal-bucket breakout-col :month)- day (assoc (lib/with-temporal-bucket breakout-col :day)- :metabase.lib.field/original-temporal-unit :month)+ day (lib/with-temporal-bucket breakout-col :day)q2 (-> query(lib/breakout month))cols (lib/orderable-columns q2)q3 (-> q2(lib/order-by (first cols))(lib/replace-clause (first (lib/breakouts q2)) day))]- (is (= (get-in q3 [:stages 0 :breakout 0 1 :metabase.lib.field/original-temporal-unit])- (get-in q3 [:stages 0 :order-by 0 2 1 :metabase.lib.field/original-temporal-unit])))))))+ (is (= (get-in q3 [:stages 0 :breakout 0 1 :temporal-unit])+ (get-in q3 [:stages 0 :order-by 0 2 1 :temporal-unit])+ :day))))))(deftest ^:parallel empty-first-stage-gets-merged-test(testing "issue #45041"