The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-2688] refactor find-stale-query to core multimethod (#76649)
YB15c5b59fe742
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 4546f321cf3..7e12acd0841 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -702,6 +702,7 @@audit-appcollectionsconfig+ embeddingeventslibmodels@@ -712,6 +713,7 @@query-processorsearchsettings+ stalenesssyncutil}:model-exports #{:model/Dashboard@@ -1718,6 +1720,7 @@content-verificationdashboardsdocuments+ embeddingeventsgraphlib@@ -1733,6 +1736,8 @@query-processorrequestsearch+ settings+ stalenesssyncutilview-log@@ -2197,6 +2202,10 @@util}:model-imports #{:model/AuthIdentity :model/PermissionsGroupMembership}}+ staleness+ {:team "UX West"+ :api #{metabase.staleness.core}}+startup{:team "DevEx":api #{metabase.startup.core}@@ -3557,11 +3566,11 @@:uses #{analyticsapicollections- embeddingpremium-featuresqueriesrequestsettings+ stalenessutil}:model-imports #{:model/Card :model/Collection :model/Dashboard}}diff --git a/enterprise/backend/src/metabase_enterprise/stale/impl.clj b/enterprise/backend/src/metabase_enterprise/stale/impl.cljindex f3e8c46a857..fd42830a6a4 100644--- a/enterprise/backend/src/metabase_enterprise/stale/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/impl.clj@@ -1,8 +1,6 @@(ns metabase-enterprise.stale.impl(:require- [metabase.embedding.settings :as embed.settings]- [metabase.settings.core :as setting]- [metabase.util.honey-sql-2 :as h2x]+ [metabase.staleness.core :as staleness][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -15,88 +13,21 @@[:limit [:maybe {:doc "The limit for pagination."} :int]][:offset [:maybe {:doc "The offset for pagination."} :int]][:sort-column [:enum {:doc "The column to sort by."} :name :last_used_at]]- [:sort-direction [:enum {:doc "The direction to sort by."} :asc :desc]]])--(defmulti ^:private find-stale-query- "Find stale content of a given model type."- {:arglists '([model args])}- (fn [model _args] model))--(defmethod find-stale-query :model/Card- [_model args]- {:select [:report_card.id- [(h2x/literal "Card") :model]- [:report_card.name :name]- :last_used_at]- :from :report_card- :left-join [:moderation_review [:and- [:= :moderation_review.moderated_item_id :report_card.id]- [:= :moderation_review.moderated_item_type (h2x/literal "card")]- [:= :moderation_review.most_recent true]- [:= :moderation_review.status (h2x/literal "verified")]]- :pulse_card [:= :pulse_card.card_id :report_card.id]- :pulse [:and- [:= :pulse_card.pulse_id :pulse.id]- [:= :pulse.archived false]]- :sandboxes [:= :sandboxes.card_id :report_card.id]- :collection [:= :collection.id :report_card.collection_id]]- :where [:and- [:= :sandboxes.id nil]- [:= :pulse.id nil]- [:= :moderation_review.id nil]- [:= :report_card.archived false]- [:<= :report_card.last_used_at (-> args :cutoff-date)]- ;; find things only in regular collections, not the `instance-analytics` collection.- [:= :collection.type nil]- (when (embed.settings/some-embedding-enabled?)- [:= :report_card.enable_embedding false])- (when (setting/get :enable-public-sharing)- [:= :report_card.public_uuid nil])- [:or- (when (contains? (:collection-ids args) nil)- [:is :report_card.collection_id nil])- [:in :report_card.collection_id (-> args :collection-ids)]]]})--(defmethod find-stale-query :model/Dashboard- [_model args]- {:select [:report_dashboard.id- [(h2x/literal "Dashboard") :model]- [:report_dashboard.name :name]- [:last_viewed_at :last_used_at]]- :from :report_dashboard- :left-join [:pulse [:and- [:= :pulse.archived false]- [:= :pulse.dashboard_id :report_dashboard.id]]- :collection [:= :collection.id :report_dashboard.collection_id]- :moderation_review [:and- [:= :moderation_review.moderated_item_id :report_dashboard.id]- [:= :moderation_review.moderated_item_type (h2x/literal "dashboard")]- [:= :moderation_review.most_recent true]- [:= :moderation_review.status (h2x/literal "verified")]]]- :where [:and- [:= :pulse.id nil]- [:= :moderation_review.id nil]- [:= :report_dashboard.archived false]- [:<= :report_dashboard.last_viewed_at (-> args :cutoff-date)]- ;; find things only in regular collections, not the `instance-analytics` collection.- [:= :collection.type nil]- (when (embed.settings/some-embedding-enabled?)- [:= :report_dashboard.enable_embedding false])- (when (setting/get :enable-public-sharing)- [:= :report_dashboard.public_uuid nil])- [:or- (when (contains? (:collection-ids args) nil)- [:is :report_dashboard.collection_id nil])- [:in :report_dashboard.collection_id (-> args :collection-ids)]]]})+ [:sort-direction [:enum {:doc "The direction to sort by."} :asc :desc]]+ [:models {:optional true} [:set {:doc "The set of models to search for stale content."} :keyword]]])(defn- sort-column [column](case column:name :%lower.name:last_used_at :last_used_at))-(defn- queries [args]- (for [model [:model/Card :model/Dashboard]]- (find-stale-query model args)))+(defn- queries [{:keys [models] :or {models #{:model/Card :model/Dashboard}} :as args}]+ ;; Ensure each model's namespace is loaded so its `find-stale-query` method is registered before we+ ;; dispatch — passing the `:model/X` keyword alone does not trigger Toucan model resolution.+ (run! t2/resolve-model models)+ (when-let [unsupported (seq (remove #(get-method staleness/find-stale-query %) models))]+ (throw (ex-info (str "No staleness method registered for: " (vec unsupported)) {})))+ (map #(staleness/find-stale-query % args) models))(mu/defn ^:private rows-query [{:keys [limit offset] :as args} :- FindStaleContentArgs](cond-> {:select [:id :model]diff --git a/enterprise/backend/test/metabase_enterprise/stale/impl_test.clj b/enterprise/backend/test/metabase_enterprise/stale/impl_test.cljindex 2816fa15891..c10b3fbcb5c 100644--- a/enterprise/backend/test/metabase_enterprise/stale/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/stale/impl_test.clj@@ -386,6 +386,30 @@:sort-column :name:sort-direction :asc}))))))+(deftest models-arg-selects-which-models-are-searched+ (with-stale-items [:model/Card {card-id :id} {:name "A card" :collection_id nil}+ :model/Dashboard {dash-id :id} {:name "B dash" :collection_id nil}]+ (let [base {:collection-ids #{nil}+ :cutoff-date (date-months-ago 6)+ :limit 10+ :offset 0+ :sort-column :name+ :sort-direction :asc}+ rows #(:rows (stale/find-candidates %))]+ (testing "default (no :models) searches Card + Dashboard"+ (is (= [{:id card-id :model :model/Card}+ {:id dash-id :model :model/Dashboard}]+ (rows base))))+ (testing "explicit :models restricts the search to the given models"+ (is (= [{:id card-id :model :model/Card}]+ (rows (assoc base :models #{:model/Card}))))+ (is (= [{:id dash-id :model :model/Dashboard}]+ (rows (assoc base :models #{:model/Dashboard})))))+ (testing "a model with no registered staleness method fails loud (no silent drop)"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"No staleness method registered for"+ (rows (assoc base :models #{:model/Card :model/Collection}))))))))+(deftest things-that-are-already-archived-do-not-appear(mt/with-temp [:model/Collection {col-id :id} {}:model/Dashboard _ (stale-dashboard {:name "A"diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex a60a1dd2949..21bef96eb25 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -143,6 +143,7 @@setupslackbotsso+ stalenessstartupsystemtaskdiff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 347c3cfaf8b..cbef429b98c 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -11,6 +11,7 @@[metabase.dashboards.models.dashboard-card :as dashboard-card][metabase.dashboards.models.dashboard-tab :as dashboard-tab][metabase.dashboards.schema :as dashboards.schema]+ [metabase.embedding.settings :as embed.settings][metabase.events.core :as events][metabase.lib.core :as lib][metabase.models.interface :as mi]@@ -23,6 +24,8 @@[metabase.queries.core :as queries][metabase.query-processor.metadata :as qp.metadata][metabase.search.core :as search]+ [metabase.settings.core :as setting]+ [metabase.staleness.core :as staleness][metabase.util :as u][metabase.util.embed :refer [maybe-populate-initially-published-at]][metabase.util.honey-sql-2 :as h2x]@@ -533,3 +536,35 @@[:= :mr.moderated_item_type "dashboard"][:= :mr.moderated_item_id :this.id][:= :mr.most_recent true]]]}})++(defmethod staleness/find-stale-query :model/Dashboard+ [_model args]+ {:select [:report_dashboard.id+ [(h2x/literal "Dashboard") :model]+ [:report_dashboard.name :name]+ [:last_viewed_at :last_used_at]]+ :from :report_dashboard+ :left-join [:pulse [:and+ [:= :pulse.archived false]+ [:= :pulse.dashboard_id :report_dashboard.id]]+ :collection [:= :collection.id :report_dashboard.collection_id]+ :moderation_review [:and+ [:= :moderation_review.moderated_item_id :report_dashboard.id]+ [:= :moderation_review.moderated_item_type (h2x/literal "dashboard")]+ [:= :moderation_review.most_recent true]+ [:= :moderation_review.status (h2x/literal "verified")]]]+ :where [:and+ [:= :pulse.id nil]+ [:= :moderation_review.id nil]+ [:= :report_dashboard.archived false]+ [:<= :report_dashboard.last_viewed_at (-> args :cutoff-date)]+ ;; find things only in regular collections, not the `instance-analytics` collection.+ [:= :collection.type nil]+ (when (embed.settings/some-embedding-enabled?)+ [:= :report_dashboard.enable_embedding false])+ (when (setting/get :enable-public-sharing)+ [:= :report_dashboard.public_uuid nil])+ [:or+ (when (contains? (:collection-ids args) nil)+ [:is :report_dashboard.collection_id nil])+ [:in :report_dashboard.collection_id (-> args :collection-ids)]]]})diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 9fd9656e098..1a50efd84f0 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -15,6 +15,7 @@[metabase.config.core :as config][metabase.content-verification.core :as moderation][metabase.dashboards.autoplace :as autoplace]+ [metabase.embedding.settings :as embed.settings][metabase.events.core :as events][metabase.graph.core :as graph][metabase.lib-be.core :as lib-be]@@ -41,6 +42,8 @@[metabase.queries.schema :as queries.schema][metabase.query-permissions.core :as query-perms][metabase.search.core :as search]+ [metabase.settings.core :as setting]+ [metabase.staleness.core :as staleness][metabase.util :as u][metabase.util.embed :refer [maybe-populate-initially-published-at]][metabase.util.honey-sql-2 :as h2x]@@ -1560,3 +1563,38 @@(search/define-spec "metric"(-> (base-search-spec) (sql.helpers/where [:= :this.type "metric"])))++(defmethod staleness/find-stale-query :model/Card+ [_model args]+ {:select [:report_card.id+ [(h2x/literal "Card") :model]+ [:report_card.name :name]+ :last_used_at]+ :from :report_card+ :left-join [:moderation_review [:and+ [:= :moderation_review.moderated_item_id :report_card.id]+ [:= :moderation_review.moderated_item_type (h2x/literal "card")]+ [:= :moderation_review.most_recent true]+ [:= :moderation_review.status (h2x/literal "verified")]]+ :pulse_card [:= :pulse_card.card_id :report_card.id]+ :pulse [:and+ [:= :pulse_card.pulse_id :pulse.id]+ [:= :pulse.archived false]]+ :sandboxes [:= :sandboxes.card_id :report_card.id]+ :collection [:= :collection.id :report_card.collection_id]]+ :where [:and+ [:= :sandboxes.id nil]+ [:= :pulse.id nil]+ [:= :moderation_review.id nil]+ [:= :report_card.archived false]+ [:<= :report_card.last_used_at (-> args :cutoff-date)]+ ;; find things only in regular collections, not the `instance-analytics` collection.+ [:= :collection.type nil]+ (when (embed.settings/some-embedding-enabled?)+ [:= :report_card.enable_embedding false])+ (when (setting/get :enable-public-sharing)+ [:= :report_card.public_uuid nil])+ [:or+ (when (contains? (:collection-ids args) nil)+ [:is :report_card.collection_id nil])+ [:in :report_card.collection_id (-> args :collection-ids)]]]})diff --git a/src/metabase/staleness/core.clj b/src/metabase/staleness/core.cljnew file mode 100644index 00000000000..feea80b3c73--- /dev/null+++ b/src/metabase/staleness/core.clj@@ -0,0 +1,19 @@+(ns metabase.staleness.core+ "Central staleness contract. Each stale-eligible model defines its own `find-stale-query` method+ in the model's own namespace. Consumers (the EE stale module's `find-candidates`) enumerate an+ explicit set of models and call this multimethod per model.++ Kept dependency-light (no model requires) so model namespaces can require it without creating a+ load cycle.")++(defmulti find-stale-query+ "Return a HoneySQL map selecting stale candidates for `model`:++ {:select [<id> [\"Model\" :model] [<name> :name] [<recency-ts> :last_used_at]] :from … :where …}++ `args` is `{:collection-ids (:all | #{int|nil}), :cutoff-date local-date}` (the same+ `FindStaleContentArgs` map the EE stale module threads through). The returned maps are+ UNION-ALL'd by the caller, so every method MUST select the same column shape:+ `id`, `model`, `name`, `last_used_at`."+ {:arglists '([model args])}+ (fn [model _args] model))diff --git a/test/metabase/dashboards/models/dashboard_test.clj b/test/metabase/dashboards/models/dashboard_test.cljindex 59049a14280..e76d3d2a9ac 100644--- a/test/metabase/dashboards/models/dashboard_test.clj+++ b/test/metabase/dashboards/models/dashboard_test.clj@@ -9,6 +9,8 @@[metabase.permissions.core :as perms][metabase.pulse.models.pulse-channel-test :as pulse-channel-test][metabase.queries.models.parameter-card :as parameter-card]+ [metabase.stale-test :as stale-test]+ [metabase.staleness.core :as staleness][metabase.test :as mt][metabase.test.data.users :as test.users][metabase.test.util :as tu]@@ -417,6 +419,32 @@[:id [:= (:id card)]]]]]]]])}}(-> dash (t2/hydrate :resolved-params) :resolved-params)))))+(deftest find-stale-query-test+ (testing "the Dashboard `find-stale-query` method selects stale dashboards and applies the model's own exclusions"+ (mt/with-temp [:model/Collection {col-id :id} {}+ :model/Dashboard {stale-id :id} (stale-test/stale-dashboard {:name "stale" :collection_id col-id})+ :model/Dashboard {fresh-id :id} {:name "fresh" :collection_id col-id+ :last_viewed_at (stale-test/datetime-months-ago 1)}+ :model/Dashboard {archived-id :id} (stale-test/stale-dashboard {:name "archived" :collection_id col-id+ :archived true})]+ (let [stale-ids (fn [] (set (map :id (t2/query (staleness/find-stale-query+ :model/Dashboard+ {:collection-ids #{col-id}+ :cutoff-date (stale-test/date-months-ago 6)})))))]+ (testing "a stale, unarchived dashboard is returned; recent and archived dashboards are not"+ (let [ids (stale-ids)]+ (is (contains? ids stale-id))+ (is (not (contains? ids fresh-id)))+ (is (not (contains? ids archived-id)))))+ (testing "a publicly shared dashboard is excluded only when public sharing is enabled"+ (mt/with-temp [:model/Dashboard {public-id :id} (stale-test/stale-dashboard+ {:name "public" :collection_id col-id+ :public_uuid (str (random-uuid))})]+ (tu/with-temporary-setting-values [enable-public-sharing false]+ (is (contains? (stale-ids) public-id)))+ (tu/with-temporary-setting-values [enable-public-sharing true]+ (is (not (contains? (stale-ids) public-id))))))))))+(deftest ^:parallel hydrate-resolved-params-model-test(mt/with-temp[:model/Dashboard dash {:parameters [{:name "Category Name"diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex ff64f661b58..82b1cd6e53f 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -20,6 +20,8 @@[metabase.query-processor.card-test :as qp.card-test][metabase.query-processor.preprocess :as qp.preprocess][metabase.search.ingestion :as search.ingestion]+ [metabase.stale-test :as stale-test]+ [metabase.staleness.core :as staleness][metabase.test :as mt][metabase.test.util :as tu][metabase.util :as u]@@ -1634,3 +1636,29 @@(let [updated-card (t2/select-one :model/Card :id (:id question))](is (= db2-id (get-in updated-card [:dataset_query :database])))(is (= db2-id (:database_id updated-card))))))))++(deftest find-stale-query-test+ (testing "the Card `find-stale-query` method selects stale cards and applies the model's own exclusions"+ (mt/with-temp [:model/Collection {col-id :id} {}+ :model/Card {stale-id :id} (stale-test/stale-card {:name "stale" :collection_id col-id})+ :model/Card {fresh-id :id} {:name "fresh" :collection_id col-id+ :last_used_at (stale-test/datetime-months-ago 1)}+ :model/Card {archived-id :id} (stale-test/stale-card {:name "archived" :collection_id col-id+ :archived true})]+ (let [stale-ids (fn [] (set (map :id (t2/query (staleness/find-stale-query+ :model/Card+ {:collection-ids #{col-id}+ :cutoff-date (stale-test/date-months-ago 6)})))))]+ (testing "a stale, unarchived card is returned; recent and archived cards are not"+ (let [ids (stale-ids)]+ (is (contains? ids stale-id))+ (is (not (contains? ids fresh-id)))+ (is (not (contains? ids archived-id)))))+ (testing "a publicly shared card is excluded only when public sharing is enabled"+ (mt/with-temp [:model/Card {public-id :id} (stale-test/stale-card+ {:name "public" :collection_id col-id+ :public_uuid (str (random-uuid))})]+ (tu/with-temporary-setting-values [enable-public-sharing false]+ (is (contains? (stale-ids) public-id)))+ (tu/with-temporary-setting-values [enable-public-sharing true]+ (is (not (contains? (stale-ids) public-id))))))))))diff --git a/test/metabase/staleness/core_test.clj b/test/metabase/staleness/core_test.cljnew file mode 100644index 00000000000..d1a9f74cdf1--- /dev/null+++ b/test/metabase/staleness/core_test.clj@@ -0,0 +1,9 @@+(ns metabase.staleness.core-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.staleness.core :as staleness]))++(deftest unregistered-model-throws+ (testing "dispatching a model with no registered staleness method throws"+ (is (thrown? IllegalArgumentException+ (staleness/find-stale-query :model/DefinitelyNotAModel {})))))