Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Re-derive the result metadata of MBQL cards from the dependency graph (#84475)

Alexander Polyankin3e8f2152034f
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 4a26230229e..69ffb6cfb88 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3906,6 +3906,7 @@
lib.metadata
lib.metric
lib.schema
+ lib.types
measures
models
native-query-snippets
@@ -3937,6 +3938,7 @@
:model/Document
:model/Field
:model/FieldUserSettings
+ :model/FieldValues
:model/Measure
:model/NativeQuerySnippet
:model/QueryAction
@@ -4427,7 +4429,6 @@
app-db
appearance
collections
- dependencies
lib
lib.schema
logger
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 1d4b2a407f4..4523030f1e2 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -25,7 +25,7 @@
:discouraged-java-method 4
:duplicate-require 1
:main-without-gen-class 1
- :metabase/discourage-dynamic-vars 325
+ :metabase/discourage-dynamic-vars 326
:metabase/discourage-millis-duration 1
:metabase/modules 23
:metabase/namespace-name 3
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj
new file mode 100644
index 00000000000..c7a44f37411
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj
@@ -0,0 +1,144 @@
+(ns metabase-enterprise.dependencies.analysis-finding
+ "Analyzes the entities whose AnalysisFindingStatus a write marked stale, recording what each one's query no longer finds."
+ (:require
+ [metabase-enterprise.dependencies.analysis :as deps.analysis]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.schema :as dependencies.schema]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defmulti ^:private entity-db-id
+ "The database ID of `entity`."
+ {:arglists '([entity])}
+ t2/model)
+
+(defmethod entity-db-id :default
+ [entity]
+ (:database_id entity))
+
+(defmethod entity-db-id :model/Transform
+ [entity]
+ (or (some-> entity :source :query lib/database-id)
+ ;; non-query transforms (e.g. Python) have no query to read the database from
+ (:source_database_id entity)))
+
+(defmethod entity-db-id :model/Segment
+ [entity]
+ (some-> entity :definition lib/database-id))
+
+(defmulti ^:private pre-analysis-errors
+ "The set of errors of `entity` found without running the analysis, such as an orphan Transform's, which when not
+ empty are recorded instead of running it."
+ {:arglists '([entity])}
+ t2/model)
+
+(defmethod pre-analysis-errors :default [_] nil)
+
+(defmethod pre-analysis-errors :model/Transform
+ [{db-id :source_database_id id :id}]
+ (when (nil? db-id)
+ #{(-> (lib/validation-exception-error "Source database for this transform has been deleted.")
+ (assoc :source-entity-type :transform :source-entity-id id))}))
+
+(mu/defn upsert-analysis!
+ "Analyze the Toucan entity `entity` and insert or replace its AnalysisFinding, reusing the caller's metadata provider
+ cache when it has one."
+ [entity :- [:fn {:error/message "a Toucan instance"} t2/instance?]]
+ (lib-be/with-metadata-provider-cache
+ (let [model (t2/model entity)
+ entity-type (deps.dependency-types/model->dependency-type model)
+ entity-id (:id entity)]
+ (if-let [errors (not-empty (pre-analysis-errors entity))]
+ (models.analysis-finding/upsert-analysis! entity-type entity-id false errors)
+ (if-let [db-id (entity-db-id entity)]
+ (let [mp (lib-be/application-database-metadata-provider db-id)
+ results (try (deps.analysis/check-entity mp entity-type entity-id)
+ (catch Exception e
+ (log/errorf "Error analyzing entity: %s" (ex-message e))
+ #{(lib/validation-exception-error (.getMessage e))}))
+ success (empty? results)]
+ (models.analysis-finding/upsert-analysis! entity-type entity-id success results))
+ (models.analysis-finding/upsert-analysis!
+ entity-type entity-id false
+ #{(lib/validation-exception-error "Could not resolve a database for this entity.")}))))))
+
+(mu/defn- analyze-entity!
+ "Analyze `entity`, recording a failure as a failed finding before rethrowing it."
+ [entity :- [:fn {:error/message "a Toucan instance"} t2/instance?]]
+ (try (upsert-analysis! entity)
+ (catch Exception e
+ (log/errorf "Analyzing entity %s %s failed: %s"
+ (t2/model entity) (:id entity) (ex-message e))
+ (try
+ (models.analysis-finding/upsert-analysis!
+ (deps.dependency-types/model->dependency-type (t2/model entity)) (:id entity) false
+ #{(lib/validation-exception-error (ex-message e))})
+ (catch Exception record-e
+ (log/errorf "Recording the failure of entity %s %s failed: %s"
+ (t2/model entity) (:id entity) (ex-message record-e))))
+ (throw e))))
+
+(def analyzable-entities
+ "Entities for which we can compute analysis findings."
+ #{:card :transform :segment})
+
+(mu/defn- remove-deleted-statuses! :- nat-int?
+ "Delete the AnalysisFindingStatuses of up to `batch-size` deleted entities of each analyzable type, returning how
+ many it deleted."
+ [batch-size :- pos-int?]
+ (reduce + (for [entity-type (sort analyzable-entities)
+ :let [ids (dependencies.db/analysis-finding-deleted-entity-ids entity-type batch-size)]]
+ (if (seq ids)
+ (dependencies.db/delete-analysis-finding-statuses! entity-type ids)
+ 0))))
+
+(mu/defn- take-pending-statuses! :- [:map-of ::deps.dependency-types/dependency-types
+ [:sequential ::dependencies.schema/pending-status]]
+ "Up to `batch-size` analyzable entities whose AnalysisFindingStatus counts a write, is missing or is outdated, as
+ `{entity-type statuses}`, each with the `stale` count the job clears afterwards."
+ [batch-size :- pos-int?]
+ (reduce (fn [taken entity-type]
+ (let [remaining (- batch-size (reduce + (map count (vals taken))))
+ statuses (when (pos? remaining)
+ (dependencies.db/take-pending-analysis-finding-statuses!
+ entity-type remaining models.analysis-finding/*current-analysis-finding-version*
+ models.dependency/current-dependency-analysis-version))]
+ (cond-> taken
+ (seq statuses) (assoc entity-type statuses))))
+ {}
+ (sort analyzable-entities)))
+
+(mu/defn analyze-batch! :- nat-int?
+ "Analyze up to `batch-size` analyzable entities whose AnalysisFindingStatus counts a write, is missing or is outdated,
+ clearing each one's `stale` count unless a write raised it meanwhile, or counting its failure, returning how many it
+ took."
+ [batch-size :- pos-int?]
+ ;; The cache spans the select too: reading an entity attaches a `MetadataProvider` to its query
+ ;; (see `metabase.lib-be.models.transforms/transform-query`), so selecting outside the cache gives every entity in
+ ;; the batch a private one.
+ (let [removed (remove-deleted-statuses! batch-size)]
+ (lib-be/with-metadata-provider-cache
+ (let [taken (take-pending-statuses! batch-size)]
+ (doseq [[entity-type statuses] taken
+ :let [stale (into {} (map (juxt :entity_id :stale)) statuses)]
+ entity (dependencies.db/entities entity-type (set (keys stale)))]
+ (try
+ (analyze-entity! entity)
+ (dependencies.db/clear-analysis-finding-status!
+ entity-type (:id entity) (stale (:id entity)) models.analysis-finding/*current-analysis-finding-version*)
+ (catch Throwable e
+ (when (dependencies.db/record-analysis-finding-failure! entity-type (:id entity)
+ (deps.settings/dependency-max-failures))
+ (log/warnf "Giving up on analyzing %s %s until it changes" (name entity-type) (:id entity)))
+ (when-not (instance? Exception e)
+ (throw e)))))
+ (+ removed (reduce + (map count (vals taken))))))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 5800c7240e7..c81fb4f9816 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -6,6 +6,7 @@
[metabase-enterprise.dependencies.models.analysis-finding-error :as analysis-finding-error]
[metabase-enterprise.dependencies.models.dependency :as dependency]
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.schema :as deps.schema]
[metabase-enterprise.sandbox.schema]
[metabase.actions.schema :as actions.schema]
[metabase.api.common :as api]
@@ -53,7 +54,13 @@
:measure [:name :description :created_at :creator :creator_id :table :table_id]
:action [:name :description :type :model_id :parameters :created_at :creator :creator_id :collection :collection_id]})
-(defn- format-subentity [entity]
+(mr/def ::data-value
+ [:schema {::mr/deliberately-open true, :description "any column or hydrated value of a dependency-tracked entity"}
+ :any])
+
+(mu/defn- format-subentity :- ::data-value
+ "`entity` slimmed to the keys the response shows when it is a hydrated Collection, Dashboard, or Document."
+ [entity :- ::data-value]
(case (t2/model entity)
:model/Collection (select-keys entity [:id :name :authority_level :is_personal :namespace :location])
:model/Dashboard (select-keys entity [:id :name])
@@ -65,16 +72,32 @@
[:enum :table :snippet :transform :dashboard :document :sandbox :segment :question :model :metric :measure :action]
::deps.dependency-types/entity-id])
+(mr/def ::normalized-finding-error
+ "The slim `{:type :detail}` shape [[normalize-finding-error]] produces from an AnalysisFindingError."
+ [:map {:closed true}
+ [:type ::lib.schema.validate/validate-error-type]
+ [:detail {:optional true} [:maybe :string]]])
+
(mr/def ::dependent-error
"One entry of `::base-entity`'s `:dependents_errors`: either the slim `{:type :detail}` shape
[[normalize-finding-error]] produces for an entity's own errors (`node-errors`), or the raw AnalysisFindingError row
`node-downstream-errors` keeps so callers can tell which downstream entity each error belongs to."
[:or
- [:map {:closed true}
- [:type ::lib.schema.validate/validate-error-type]
- [:detail {:optional true} [:maybe :string]]]
+ ::normalized-finding-error
::analysis-finding-error/analysis-finding-error])
+(mr/def ::nodes-by-type
+ "Entity ids grouped by entity type."
+ [:map-of ::deps.dependency-types/dependency-types [:sequential ::deps.dependency-types/entity-id]])
+
+(mr/def ::node->usages
+ "The dependent counts of each node."
+ [:map-of ::deps.schema/node ::usages])
+
+(mr/def ::node->errors
+ "The errors of each node."
+ [:map-of ::deps.schema/node [:set ::dependent-error]])
+
(mr/def ::base-entity
[:map
[:id pos-int?]
@@ -83,10 +106,10 @@
[:dependents_count [:maybe [:ref ::usages]]]
[:dependents_errors {:optional true} [:set [:ref ::dependent-error]]]])
-(defn- fields-for [entity-key]
- ;; these specs should really use something like
- #_[:data [:select-keys [:ref :blah/table] (entity-keys :table)]]
- ;; but :select-keys seems to mess up open-api spec generation
+(mu/defn- fields-for :- [:cat [:= :map] [:* [:tuple :keyword [:map {:closed true} [:optional [:= true]]] [:= :any]]]]
+ "An open map schema with every key of `entity-key`'s `entity-keys` optional; not a `:select-keys` schema, which
+ breaks the OpenAPI spec generation."
+ [entity-key :- (into [:enum] (keys entity-keys))]
(into [:map]
(map (fn [key] [key {:optional true} :any]))
(entity-keys entity-key)))
@@ -174,6 +197,26 @@
[:measure [:ref ::measure-entity]]
[:action [:ref ::action-entity]]])
+(mr/def ::entity-data
+ "The `:data` of an `::entity` as the helpers that filter and sort entities read it: the keys of `entity-keys`, each
+ optional."
+ (into [:map {:closed true}]
+ (map (fn [k]
+ [k {:optional true} (case k
+ (:name :display_name) [:maybe :string]
+ :view_count [:maybe :int]
+ ::data-value)]))
+ (distinct (mapcat val entity-keys))))
+
+(mr/def ::entity-node
+ "An `::entity`, or a `::broken-dependent-entity`, as the helpers that filter and sort entities read it."
+ [:map {:closed true}
+ [:id ::deps.dependency-types/entity-id]
+ [:type ::deps.dependency-types/dependency-types]
+ [:data ::entity-data]
+ [:dependents_count {:optional true} [:maybe [:ref ::usages]]]
+ [:dependents_errors {:optional true} [:set [:ref ::dependent-error]]]])
+
(def ^:private Entity
"Any dependency-tracked entity `entity-value` accepts, hydrated the way `hydrate-entities` hydrates it."
[:or
@@ -191,12 +234,8 @@
(mu/defn- entity-value :- ::entity
[entity-type :- ::deps.dependency-types/dependency-types
{:keys [id] :as entity} :- Entity
- usages :- [:maybe [:map-of
- [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]
- ::usages]]
- errors :- [:maybe [:map-of
- [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]
- [:set ::dependent-error]]]]
+ usages :- [:maybe ::node->usages]
+ errors :- [:maybe ::node->errors]]
(cond-> {:id id
:type entity-type
:data (-> (select-keys entity (entity-keys entity-type))
@@ -233,34 +272,55 @@
:measure [:id :name :description :created_at :creator_id :table_id]
:action [:id :name :description :type :model_id :parameters :created_at :creator_id :collection_id]})
-(defn- current-user-visibility
+(mr/def ::include-archived-items
+ [:enum :exclude :all :only])
+
+(mr/def ::visible-opts
+ "The `:visible` filter-spec opts consumed by `metabase-enterprise.dependencies.db`."
+ [:map {:closed true}
+ [:user-id ::lib.schema.id/user]
+ [:is-superuser? {:optional true} [:maybe :boolean]]
+ [:is-data-analyst? {:optional true} [:maybe :boolean]]
+ [:include-archived-items {:optional true} ::include-archived-items]])
+
+(mu/defn- current-user-visibility :- ::visible-opts
"The current user, as the `:visible` filter-spec opts consumed by `metabase-enterprise.dependencies.db`.
Analyst-wide visibility comes with the role rather than a permissions graph, so it pauses while the
`advanced-permissions` feature is unavailable (see [[api/entitled-data-analyst?]])."
- [{:keys [include-archived-items]}]
+ [{:keys [include-archived-items]} :- [:maybe [:map {:closed true}
+ [:include-archived-items {:optional true}
+ [:maybe ::include-archived-items]]]]]
(cond-> {:user-id api/*current-user-id*
:is-superuser? api/*is-superuser?*
:is-data-analyst? (api/entitled-data-analyst?)}
include-archived-items (assoc :include-archived-items include-archived-items)))
-(defn- readable-graph-dependencies
+(mu/defn- readable-graph-dependencies :- ::graph/graph
+ "The upstream dependency graph, restricted to the entities the current user can see."
([]
(readable-graph-dependencies nil))
- ([opts]
+ ([opts :- [:maybe [:map {:closed true}
+ [:include-archived-items {:optional true} [:maybe ::include-archived-items]]]]]
(dependency/filtered-graph-dependencies {:visible (current-user-visibility opts)})))
-(defn- readable-graph-dependents
+(mu/defn- readable-graph-dependents :- ::graph/graph
+ "The downstream dependents graph, restricted to the entities the current user can see, and to broken ones when
+ `broken`."
([]
(readable-graph-dependents nil))
- ([{:keys [include-archived-items broken] :or {include-archived-items :exclude}}]
+ ([{:keys [include-archived-items broken] :or {include-archived-items :exclude}}
+ :- [:maybe [:map {:closed true}
+ [:include-archived-items {:optional true} [:maybe ::include-archived-items]]
+ [:broken {:optional true} [:maybe :boolean]]]]]
(dependency/filtered-graph-dependents
(cond-> {:visible (current-user-visibility {:include-archived-items include-archived-items})}
broken (assoc :broken? true)))))
-(defn- node-usages
- "Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`. "
- [graph nodes]
+(mu/defn- node-usages :- ::node->usages
+ "Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`."
+ [graph :- ::graph/graph
+ nodes :- [:sequential ::deps.schema/node]]
(let [children-map (graph/children-of graph nodes)
all-cards (->> (vals children-map)
(apply concat)
@@ -279,18 +339,19 @@
{dependency-type 1})))
(apply merge-with +))))))
-(defn- normalize-finding-error
- [{:keys [error_type error_detail]}]
+(mu/defn- normalize-finding-error :- ::normalized-finding-error
+ "The error type and detail of an AnalysisFindingError."
+ [{:keys [error_type error_detail]} :- ::analysis-finding-error/analysis-finding-error]
(cond-> {:type error_type}
error_detail (assoc :detail error_detail)))
-(defn- node-downstream-errors
+(mu/defn- node-downstream-errors :- [:maybe ::node->errors]
"Fetches errors caused by the given source entities (what downstream entities they're breaking).
Filters out errors where the analyzed entity is not visible to the current user.
Unlike `node-errors` which fetches errors on an entity, this fetches errors that
the entity is causing in other entities that depend on it. Keeps the raw AnalysisFindingError shape (unlike
`node-errors`'s normalized one) so callers can tell which downstream entity each error belongs to."
- [nodes-by-type]
+ [nodes-by-type :- ::nodes-by-type]
(letfn [(errors-by-source-type-and-id [[source-type ids]]
(when (seq ids)
(let [finding-errors (dependencies.db/finding-errors-from-sources
@@ -301,11 +362,11 @@
(into {} (mapcat errors-by-source-type-and-id))
not-empty)))
-(defn- node-errors
+(mu/defn- node-errors :- [:maybe ::node->errors]
"Fetches and normalizes AnalysisFindingErrors for the given entities.
Filters out errors where the source entity is not visible to the current user.
Returns {[entity-type entity-id] #{error-maps...}}, or nil if none."
- [nodes-by-type]
+ [nodes-by-type :- ::nodes-by-type]
(letfn [(errors-by-entity-type-and-id [[type ids]]
(when (seq ids)
(let [finding-errors (dependencies.db/finding-errors-for-entities-with-visible-sources
@@ -316,7 +377,10 @@
(into {} (mapcat errors-by-entity-type-and-id))
not-empty)))
-(defn- hydrate-entities [entity-type entities]
+(mu/defn- hydrate-entities :- [:sequential Entity]
+ "`entities` of the entity type `entity-type`, hydrated with what the dependency graph responses show."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entities :- [:sequential Entity]]
(case entity-type
:card (-> entities
(t2/hydrate :creator :dashboard :document [:collection :is_personal])
@@ -344,21 +408,27 @@
(collection.root/root-collection-with-ui-details
(dependencies.db/action-root-namespace (nil? (:model_id %)))))))))
-(defn- fetch-and-hydrate-nodes
+(mu/defn- fetch-and-hydrate-nodes :- [:map-of ::deps.schema/node Entity]
"Fetches and hydrates entities for the given nodes.
Returns a map from [entity-type entity-id] -> hydrated entity."
- [nodes-by-type]
+ [nodes-by-type :- ::nodes-by-type]
(into {}
(mapcat (fn [[entity-type entity-ids]]
(when (seq entity-ids)
(let [fields (entity-select-fields entity-type)]
- (->> (dependencies.db/instances-with-columns entity-type fields entity-ids)
+ (->> (dependencies.db/entities-with-columns entity-type fields entity-ids)
(hydrate-entities entity-type)
(map (fn [entity]
[[entity-type (:id entity)] entity])))))))
nodes-by-type))
-(defn- expanded-nodes [downstream-graph nodes {:keys [include-errors?]}]
+(mu/defn- expanded-nodes :- [:sequential ::entity]
+ "The `::entity` of each of `nodes` the current user can see, with its dependent counts from `downstream-graph`, and
+ its errors when `include-errors?`."
+ [downstream-graph :- ::graph/graph
+ nodes :- [:sequential ::deps.schema/node]
+ {:keys [include-errors?]} :- [:map {:closed true}
+ [:include-errors? :boolean]]]
(let [usages (node-usages downstream-graph nodes)
nodes-by-type (-> (group-by first nodes)
(update-vals #(map second %)))
@@ -394,8 +464,8 @@
upstream-graph (readable-graph-dependencies {:include-archived-items :all})
downstream-graph (graph/cached-graph (readable-graph-dependents))
edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))
- nodes (into (set starting-nodes)
- (graph/transitive upstream-graph starting-nodes))
+ nodes (vec (into (set starting-nodes)
+ (graph/transitive upstream-graph starting-nodes)))
edges (graph/edges-between edge-graph nodes)]
{:nodes (expanded-nodes downstream-graph nodes {:include-errors? false})
:edges edges}))
@@ -408,18 +478,18 @@
"Valid sort columns for the /graph/dependents endpoint."
#{:name :location :view-count})
-(defn- entity-name
+(mu/defn- entity-name :- [:maybe :string]
"Returns the name string for an entity based on its type. Might return nil."
- [entity]
+ [entity :- ::entity-node]
(let [data (:data entity)]
(case (:type entity)
:table (:display_name data)
:sandbox nil
(:name data))))
-(defn- entity-location
+(mu/defn- entity-location :- [:maybe :string]
"Returns the location string for an entity based on its type. Might return nil."
- [entity]
+ [entity :- ::entity-node]
(let [data (:data entity)]
(case (:type entity)
:card (or (-> data :dashboard :name)
@@ -431,27 +501,31 @@
(:segment :measure) (-> data :table :display_name)
nil)))
-(defn- string-matches-query?
+(mu/defn- string-matches-query? :- [:maybe :boolean]
"Returns true if `s` is a string and its lower-case version contains the string `query`.
`query` is expected to be lower-case."
- [s query]
+ [s :- [:maybe :string]
+ query :- :string]
(some-> s u/lower-case-en (str/includes? query)))
-(defn- entity-matches-query?
+(mu/defn- entity-matches-query? :- [:maybe :boolean]
"Returns true if `entity`'s name or location contains the string `query` (case-insensitive)."
- [entity query]
+ [entity :- ::entity-node
+ query :- :string]
(let [q (u/lower-case-en query)]
(or (string-matches-query? (entity-name entity) q)
(string-matches-query? (entity-location entity) q))))
-(defn- in-personal-collection?
+(mu/defn- in-personal-collection? :- [:maybe :boolean]
"Returns true if `entity` is in a personal collection."
- [entity]
+ [entity :- ::entity-node]
(get-in entity [:data :collection :is_personal]))
-(defn- sort-dependents
+(mu/defn- sort-dependents :- [:sequential ::entity-node]
"Sort `entities` by `sort-column` in `sort-direction`."
- [entities sort-column sort-direction]
+ [entities :- [:sequential ::entity-node]
+ sort-column :- (into [:enum] dependents-sort-columns)
+ sort-direction :- (into [:enum] sort-directions)]
(let [key-fn (case sort-column
:name #(some-> (entity-name %) u/lower-case-en)
:location #(some-> (entity-location %) u/lower-case-en)
@@ -503,8 +577,8 @@
sort-direction :asc}} :- dependents-args]
(api/read-check (deps.dependency-types/dependency-type->model type) id)
(let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))
- nodes (-> (graph/children-of downstream-graph [[type id]])
- (get [type id]))
+ nodes (vec (-> (graph/children-of downstream-graph [[type id]])
+ (get [type id])))
dep-types-set (cond
(nil? dependent-types) deps.dependency-types/dependency-types
(sequential? dependent-types) (set dependent-types)
@@ -658,7 +732,7 @@
usages (node-usages downstream-graph all-ids)
fetch-entity (fn [entity-type entity-id]
(let [fields (entity-select-fields entity-type)]
- (dependencies.db/instance-with-columns entity-type fields entity-id)))
+ (dependencies.db/entity-with-columns entity-type fields entity-id)))
data (into []
(keep (fn [[entity-type entity-id]]
(when-let [entity (fetch-entity entity-type entity-id)]
@@ -742,10 +816,10 @@
(api.macros/defendpoint :get "/backfill-status" :- [:map
[:complete :boolean]]
- "Returns whether the dependency backfill has pending work.
- `complete` is true when there are no stale or outdated entities awaiting processing."
+ "Whether no entity awaits the computation of its dependencies, as `complete`, for superusers."
[_route-params _query-params]
- {:complete (not (deps.dependency-status/has-stale-or-outdated?))})
+ (api/check-superuser)
+ {:complete (not (deps.dependency-status/has-pending-entities?))})
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/dependencies` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index b0288749f1a..ce1e9ab49cd 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -67,7 +67,7 @@
(let [driver (:engine (lib.metadata/database query))
deps (deps.native/native-query-deps driver query)]
;; The deps are in #{{:table 7} ...} form and need conversion to ::deps.schema/upstream-deps form.
- (u/group-by ffirst (comp second first) conj #{} deps)))
+ (u/group-by ffirst (comp second first) conj #{} (remove :unresolved-table deps))))
(mu/defn- upstream-deps:query :- ::deps.schema/upstream-deps
[query :- ::lib.schema/query]
@@ -86,6 +86,7 @@
;; Modified implementation of documents.models.document/document-deps
(defn- document-deps
+ "The `[entity-type entity-id]` pairs that the smart links and card embeds of a prose-mirror `document` point at."
[{:keys [content_type] :as document}]
(when (= content_type prose-mirror/prose-mirror-content-type)
(prose-mirror/collect-ast document (fn [{:keys [type attrs]}]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
index 45d04b5c9b8..75f904983e5 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj
@@ -7,17 +7,20 @@
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.metadata-provider :as deps.provider]
[metabase-enterprise.dependencies.models.dependency :as deps.graph]
+ [metabase-enterprise.dependencies.schema :as deps.schema]
[metabase.graph.core :as graph]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.validate :as lib.schema.validate]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(mr/def ::entity-type
[:enum :card :transform :snippet :table])
@@ -29,8 +32,14 @@
[:snippet {:optional true} [:sequential ::lib.schema.metadata/native-query-snippet]]
[:table {:optional true} [:sequential ::lib.schema.metadata/table]]])
-(defn- transitive-dependents
- [& {:keys [graph updated-entities include-native?]}]
+(mu/defn- transitive-dependents :- ::deps.schema/type->ids
+ "The transitive dependents of `updated-entities` in `graph` (the app DB by default), leaving out native entities and
+ their children unless `include-native?`."
+ [& {:keys [graph updated-entities include-native?]}
+ :- [:map {:closed true}
+ [:graph {:optional true} [:maybe ::graph/graph]]
+ [:updated-entities ::updates-map]
+ [:include-native? {:optional true} [:maybe :boolean]]]]
(if include-native?
(deps.graph/transitive-dependents graph updated-entities)
(deps.graph/transitive-mbql-dependents graph updated-entities)))
@@ -66,7 +75,7 @@
:updated-entities updated-entities
:dependent-ids dependents}))))
-(mu/defn- check-query-soundness :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error]]]]
+(mu/defn- check-query-soundness :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error-with-source]]]]
"Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependents
to check that everything is still sound.
@@ -92,7 +101,10 @@
(vswap! errors assoc-in [entity-type id] bad-refs)))
@errors))
-(defn- group-by-db [deps]
+(mu/defn- group-by-db :- [:map-of ::lib.schema.id/database [:map-of ::entity-type [:set ms/PositiveInt]]]
+ "The card, table, transform, and snippet ids of `deps` grouped by the id of the Database each belongs to, with the
+ snippets copied to every Database."
+ [deps :- ::deps.schema/type->ids]
(let [by-db (volatile! {})]
(when (seq (:card deps))
(doseq [[db-id card-ids] (->> (dependencies.db/card-database-ids (:card deps))
@@ -113,7 +125,7 @@
(vswap! by-db update-vals #(assoc % :snippet (:snippet deps))))
@by-db))
-(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error]]]]
+(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error-with-source]]]]
"Given a regular `MetadataProvider`, and a map of entity types (`:card`, `:transform`, `:snippet`) to lists of
updated entities, this returns a map of `{entity-type {entity-id [bad-ref ...]}}`.
@@ -133,9 +145,9 @@
(let [valid-edits (if include-native?
edits
(into {}
- (map (fn [[entity-type instances]]
+ (map (fn [[entity-type entities]]
[entity-type (remove #(deps.graph/is-native-entity? entity-type %)
- instances)]))
+ entities)]))
edits))]
(cond
(not (some seq (vals valid-edits)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index ee424bdee36..b413983f665 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -1,6 +1,6 @@
(ns metabase-enterprise.dependencies.db
- "Application database queries for the dependencies module. Every function here is a direct Toucan 2 call with no
- additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods, and transactions."
+ "Application database queries for the dependencies module, so the rest of the module only touches `toucan2.core` for
+ model definitions, hydration methods, and transactions."
(:require
[clojure.set :as set]
[malli.util :as mut]
@@ -10,48 +10,34 @@
[metabase.collections.models.collection :as collection]
[metabase.collections.models.collection.root :as collection.root]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
+ [metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
-(def ^:private VisibleOpts
- "Opts consumed by [[visible-entities-expr]]: the current user (`:user-id`, `:is-superuser?`,
- `:is-data-analyst?`), plus whether to include archived items (`:include-archived-items`,
- default `:exclude`)."
- [:map {:closed true}
- [:user-id ::lib.schema.id/user]
- [:is-superuser? {:optional true} [:maybe :boolean]]
- [:is-data-analyst? {:optional true} [:maybe :boolean]]
- [:include-archived-items {:optional true} [:enum :exclude :all :only]]])
-
-(def ^:private EntityType
- "A dependency-type keyword, or the equivalent string. Every column these entity types are filtered against uses
- Toucan's keyword transform, which coerces either representation, so callers (and tests) sometimes pass a plain
- string instead of the keyword."
- [:or ::deps.dependency-types/dependency-types :string])
-
-(def ^:private RestrictionSpec
- "The edge-restriction filter spec consumed by [[edge-restriction-expr]]: nil for no restriction, or a map
- combining `:visible` ([[VisibleOpts]]), `:broken?`, `:types` (a set of dependency types), and/or
- `:entity-type` + `:ids` (restricting to specific entities of one type)."
- [:maybe
- [:map {:closed true}
- [:visible {:optional true} VisibleOpts]
- [:broken? {:optional true} :boolean]
- [:types {:optional true} [:set EntityType]]
- [:entity-type {:optional true} EntityType]
- [:ids {:optional true} [:sequential ::deps.dependency-types/entity-id]]]])
+(def ^:private Column
+ "A Honey SQL column keyword, or a string literal standing in for one."
+ [:or :keyword :string])
+
+(def ^:private Join
+ "A LEFT JOIN [[build-left-joins]] adds."
+ [:enum :collection :dashboard :database :document :table])
+
+(def ^:private HoneySQLForm
+ "A Honey SQL expression or query map."
+ [:or ::h2x/honeysql-expr ::h2x/honeysql-clause-opts])
;;; ------------------------------------------------ Graph edge restrictions ------------------------------------------------
;;; The dependency graph is traversed generically over every entity type the module knows about; these private
;;; helpers build the permission and type-allowlist fragments the graph and item-list queries below restrict edges
;;; and items by.
-(defn- visible-entities-expr
+(mu/defn- visible-entities-expr :- ::h2x/honeysql-expr
"Matches entities at `entity-type-field`/`entity-id-field` that are readable by the user described by `user-id`,
`is-superuser?`, and `is-data-analyst?`, honoring `include-archived-items` (`:exclude`, `:all`, or `:only`,
default `:exclude`; applies to both archived collections and archived entities).
@@ -65,12 +51,14 @@
- Table: filters by table visibility permissions. Tables are NOT filtered by active/visibility_type regardless
of `include-archived-items`, so dependencies broken by dropped or hidden tables stay visible.
- Transform: analysts can view any transform they have source view permission to."
- [entity-type-field entity-id-field {:keys [user-id is-superuser? is-data-analyst? include-archived-items]
- :or {include-archived-items :exclude}}]
+ [entity-type-field :- Column
+ entity-id-field :- Column
+ {:keys [user-id is-superuser? is-data-analyst? include-archived-items]
+ :or {include-archived-items :exclude}} :- ::dependencies.schema/visible-opts]
(into [:or]
(keep (fn [[entity-type model]]
(let [table-name (t2/table-name model)
- id-column (keyword (name table-name) "id")]
+ id-column (u/qualified-key table-name :id)]
(case model
:model/Sandbox
(when is-superuser?
@@ -100,10 +88,10 @@
{:perms/create-queries :query-builder})]}]])
(:model/Card :model/Dashboard :model/Document :model/NativeQuerySnippet :model/Action)
- (let [archived-column (keyword (name table-name) "archived")
+ (let [archived-column (u/qualified-key table-name :archived)
visible-in-root (fn [root-namespace]
(collection/visible-collection-filter-clause
- (keyword (name table-name) "collection_id")
+ (u/qualified-key table-name :collection_id)
{:include-archived-items include-archived-items
:root-namespace root-namespace}
{:current-user-id user-id
@@ -147,8 +135,8 @@
{:perms/view-data :unrestricted :perms/create-queries :query-builder})]}]]
(:model/Segment :model/Measure)
- (let [archived-column (keyword (name table-name) "archived")
- table-id-column (keyword (name table-name) "table_id")]
+ (let [archived-column (u/qualified-key table-name :archived)
+ table-id-column (u/qualified-key table-name :table_id)]
[:and
[:= entity-type-field (name entity-type)]
[:in entity-id-field
@@ -171,40 +159,48 @@
:all nil)]}]])))))
deps.dependency-types/dependency-type->model))
-(defn- broken-entities-expr
+(mu/defn- broken-entities-expr :- ::h2x/honeysql-expr
"Matches entities at `entity-type-field`/`entity-id-field` that have failed analysis (an AnalysisFinding with
`:result` false)."
- [entity-type-field entity-id-field]
+ [entity-type-field :- Column
+ entity-id-field :- Column]
(into [:or]
(keep (fn [[entity-type _model]]
[:and
[:= entity-type-field (name entity-type)]
[:in entity-id-field
^:allow-subquery
- {:select [:analyzed_entity_id]
+ {:select [:entity_id]
:from [:analysis_finding]
:where [:and
- [:= :analysis_finding.analyzed_entity_type (name entity-type)]
+ [:= :analysis_finding.entity_type (name entity-type)]
[:= :analysis_finding.result false]]}]]))
deps.dependency-types/dependency-type->model))
-(defn- allowed-entity-types-expr
+(mu/defn- allowed-entity-types-expr :- ::h2x/honeysql-expr
"Matches entities at `entity-type-field` whose type is one of `entity-types`."
- [entity-type-field entity-types]
+ [entity-type-field :- Column
+ entity-types :- [:set ::dependencies.schema/entity-type]]
(into [:or] (map (fn [entity-type] [:= entity-type-field (name entity-type)])) entity-types))
-(defn- entity-ids-expr
+(mu/defn- entity-ids-expr :- ::h2x/honeysql-expr
"Matches entities at `entity-type-field`/`entity-id-field` that are of type `entity-type` and whose id is in
`ids`."
- [entity-type-field entity-id-field {:keys [entity-type ids]}]
+ [entity-type-field :- Column
+ entity-id-field :- Column
+ {:keys [entity-type ids]} :- [:map {:closed true}
+ [:entity-type ::dependencies.schema/entity-type]
+ [:ids [:sequential ::deps.dependency-types/entity-id]]]]
[:and [:= entity-type-field (name entity-type)] [:in entity-id-field ids]])
-(defn- edge-restriction-expr
+(mu/defn- edge-restriction-expr :- [:maybe ::h2x/honeysql-expr]
"The combined `:where` fragment for `restriction-spec` (nil for no filter), which may include `:visible` (opts for
[[visible-entities-expr]]), `:broken?` (restrict to [[broken-entities-expr]]), `:types` (restrict to
[[allowed-entity-types-expr]]), or `:entity-type` + `:ids` (restrict to [[entity-ids-expr]]). Combinable
fragments are AND-ed together."
- [entity-type-field entity-id-field {:keys [visible broken? types entity-type ids]}]
+ [entity-type-field :- Column
+ entity-id-field :- Column
+ {:keys [visible broken? types entity-type ids]} :- ::dependencies.schema/edge-restriction]
(let [fragments (cond-> []
visible (conj (visible-entities-expr entity-type-field entity-id-field visible))
broken? (conj (broken-entities-expr entity-type-field entity-id-field))
@@ -228,24 +224,27 @@
[:src-id [:enum :from_entity_id :to_entity_id]]
[:dst-type [:enum :from_entity_type :to_entity_type]]
[:dst-id [:enum :from_entity_id :to_entity_id]]
- [:entity-type EntityType]
- [:entity-ids [:or [:set ::deps.dependency-types/entity-id] [:sequential ::deps.dependency-types/entity-id]]]
- [:destination-restriction RestrictionSpec]
- [:source-restriction RestrictionSpec]]]
- (t2/select :model/Dependency
- {:where (into [:and
- [:= src-type (name entity-type)]
- [:in src-id entity-ids]]
- (keep identity)
- [(edge-restriction-expr dst-type dst-id destination-restriction)
- (edge-restriction-expr src-type src-id source-restriction)])}))
+ [:entity-type ::dependencies.schema/entity-type]
+ [:entity-ids [:sequential ::deps.dependency-types/entity-id]]
+ [:destination-restriction ::dependencies.schema/edge-restriction]
+ [:source-restriction ::dependencies.schema/edge-restriction]]]
+ (into []
+ (mapcat (fn [batch]
+ (t2/select :model/Dependency
+ {:where (into [:and
+ [:= src-type (name entity-type)]
+ [:in src-id batch]]
+ (keep identity)
+ [(edge-restriction-expr dst-type dst-id destination-restriction)
+ (edge-restriction-expr src-type src-id source-restriction)])})))
+ (partition-all 1000 entity-ids)))
(mu/defn finding-errors-from-sources
"The AnalysisFindingErrors caused by any of the entities `source-entity-type` `source-entity-ids`, whose analyzed
entity is visible to the user described by `user-id`, `is-superuser?`, and `is-data-analyst?`."
- [source-entity-type :- EntityType
+ [source-entity-type :- ::dependencies.schema/entity-type
source-entity-ids :- [:sequential ::deps.dependency-types/entity-id]
- {:keys [user-id is-superuser? is-data-analyst?]} :- VisibleOpts]
+ {:keys [user-id is-superuser? is-data-analyst?]} :- ::dependencies.schema/visible-opts]
(t2/select :model/AnalysisFindingError
{:where [:and
[:= :source_entity_type (name source-entity-type)]
@@ -257,9 +256,9 @@
(mu/defn finding-errors-for-entities-with-visible-sources
"The AnalysisFindingErrors analyzing any of the entities `entity-type` `entity-ids`, excluding those whose source
entity exists but is not visible to the user described by `user-id`, `is-superuser?`, and `is-data-analyst?`."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-ids :- [:sequential ::deps.dependency-types/entity-id]
- {:keys [user-id is-superuser? is-data-analyst?]} :- VisibleOpts]
+ {:keys [user-id is-superuser? is-data-analyst?]} :- ::dependencies.schema/visible-opts]
(t2/select :model/AnalysisFindingError
{:where [:and
[:= :analyzed_entity_type (name entity-type)]
@@ -275,21 +274,48 @@
;;; `entity_type`/`entity_id`/`sort_key` rows, then page and count that union. These private helpers build the
;;; per-entity-type SELECT and the pieces (joins, extra filters, sort expression) that make it up.
-(defn- personal-root-collection-ids
+(def ^:private DependencyItemParams
+ "Params shared by [[dependency-item-ids]] and [[dependency-item-count]]."
+ [:map {:closed true}
+ [:query-type [:enum :unreferenced :breaking]]
+ [:entity-types [:sequential ::deps.dependency-types/dependency-types]]
+ [:card-types {:optional true} [:maybe [:sequential [:or :keyword :string]]]]
+ [:search-text {:optional true} [:maybe :string]]
+ [:include-personal-collections? :boolean]
+ [:sort-column {:optional true} [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]]
+ [:sort-direction [:enum :asc :desc]]
+ [:offset ms/IntGreaterThanOrEqualToZero]
+ [:limit ms/PositiveInt]
+ [:user-id ::lib.schema.id/user]
+ [:is-superuser? {:optional true} [:maybe :boolean]]
+ [:is-data-analyst? {:optional true} [:maybe :boolean]]])
+
+(def ^:private DependencyItemSelectParams
+ "The [[DependencyItemParams]] of the SELECT of one `:entity-type`."
+ (mut/assoc DependencyItemParams :entity-type ::deps.dependency-types/dependency-types))
+
+(mu/defn- personal-root-collection-ids :- [:sequential ::lib.schema.id/collection]
"The IDs of the personal Collections."
[]
- (t2/select-pks-vec :model/Collection :personal_owner_id [:not= nil] :location "/"))
+ (or (t2/select-pks-vec :model/Collection :personal_owner_id [:not= nil] :location "/") []))
-(defn action-root-namespace
+(mu/defn action-root-namespace :- [:maybe [:= :data-actions]]
"The namespace of the root collection an action without a collection sits in: the data actions root for a
`model-less?` action, and the default root for an action of a model."
- [model-less?]
+ [model-less? :- :boolean]
(when model-less?
:data-actions))
-(defn- entity-type-config
+(def ^:private EntityTypeConfig
+ "What [[entity-type-config]] returns for an entity type."
+ [:map {:closed true}
+ [:table-name [:or :keyword ::h2x/honeysql-clause-opts]]
+ [:name-column ::h2x/honeysql-expr]
+ [:location-column ::h2x/honeysql-expr]])
+
+(mu/defn- entity-type-config :- EntityTypeConfig
"The table name and name/location column expressions to select an item list row for `entity-type`."
- [entity-type]
+ [entity-type :- ::deps.dependency-types/dependency-types]
(let [root-collection (collection.root/root-collection-with-ui-details
(case entity-type
:transform :transforms
@@ -327,11 +353,15 @@
:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)]
(:segment :measure) :table.display_name)}))
-(defn- item-query-join
+(mu/defn- item-query-join :- [:map {:closed true}
+ [:join [:sequential HoneySQLForm]]
+ [:join-filter ::h2x/honeysql-expr]]
"The join and join-filter for `query-type`, restricting the joined-in dependency/finding-error rows to those
whose other-side entity passes `default-visible-restriction` (see [[edge-restriction-expr]]; this is independent of any
archived-items override applied to the item's own visibility)."
- [query-type entity-type default-visible-restriction]
+ [query-type :- [:enum :unreferenced :broken :breaking]
+ entity-type :- ::deps.dependency-types/dependency-types
+ default-visible-restriction :- ::dependencies.schema/edge-restriction]
(case query-type
:unreferenced {:join [:dependency [:and
[:= :dependency.to_entity_id :entity.id]
@@ -340,8 +370,8 @@
default-visible-restriction)]]
:join-filter [:= :dependency.id nil]}
:broken {:join [:analysis_finding [:and
- [:= :analysis_finding.analyzed_entity_id :entity.id]
- [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]
+ [:= :analysis_finding.entity_id :entity.id]
+ [:= :analysis_finding.entity_type (name entity-type)]]]
:join-filter [:= :analysis_finding.result false]}
:breaking {:join [:analysis_finding_error [:and
[:= :analysis_finding_error.source_entity_id :entity.id]
@@ -351,9 +381,9 @@
default-visible-restriction)]]
:join-filter [:!= :analysis_finding_error.id nil]}))
-(defn- location-joins-for-entity
+(mu/defn- location-joins-for-entity :- [:set Join]
"The set of join keywords needed for location-based operations on `entity-type`."
- [entity-type]
+ [entity-type :- ::deps.dependency-types/dependency-types]
(case entity-type
:card #{:collection :dashboard :document}
(:transform :snippet :dashboard :document :action) #{:collection}
@@ -361,11 +391,13 @@
:table #{:database}
#{}))
-(defn- optional-item-expr-and-joins
+(mu/defn- optional-item-expr-and-joins :- [:map {:closed true}
+ [:exprs [:sequential ::h2x/honeysql-expr]]
+ [:joins [:set Join]]]
"The extra `:where` fragments and joins for `params`' card-type, search-text, database, internal-content,
archived, and personal-collection restrictions."
- [{:keys [query-type entity-type card-types search-text include-personal-collections?]}
- {:keys [name-column location-column]}]
+ [{:keys [query-type entity-type card-types search-text include-personal-collections?]} :- DependencyItemSelectParams
+ {:keys [name-column location-column]} :- EntityTypeConfig]
(let [card-type-expr (when (and (= entity-type :card) (seq card-types))
{:expr [:in :entity.type (mapv name card-types)]
:joins #{}})
@@ -416,10 +448,16 @@
{:exprs (keep :expr results)
:joins (reduce set/union #{} (map :joins results))}))
-(defn- sort-key-expr-and-joins
+(mu/defn- sort-key-expr-and-joins :- [:map {:closed true}
+ [:sort-expr HoneySQLForm]
+ [:joins [:set Join]]]
"The sort expression and extra joins for `sort-column`, restricting the `dependents-errors`/`dependents-with-errors`
counts to finding errors whose analyzed entity passes `default-visible-restriction` (see [[edge-restriction-expr]])."
- [sort-column entity-type name-column location-column default-visible-restriction]
+ [sort-column :- [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]
+ entity-type :- ::deps.dependency-types/dependency-types
+ name-column :- ::h2x/honeysql-expr
+ location-column :- ::h2x/honeysql-expr
+ default-visible-restriction :- ::dependencies.schema/edge-restriction]
(case sort-column
:location {:sort-expr location-column
:joins (location-joins-for-entity entity-type)}
@@ -446,10 +484,11 @@
{:sort-expr name-column
:joins #{}}))
-(defn- build-left-joins
+(mu/defn- build-left-joins :- [:sequential HoneySQLForm]
"`base-join` with the LEFT JOINs needed for `joins` (a set of `:database`, `:collection`, `:dashboard`,
`:document`, and/or `:table`) appended."
- [base-join joins]
+ [base-join :- [:sequential HoneySQLForm]
+ joins :- [:set Join]]
(cond-> base-join
(:database joins) (conj [:metabase_database :database] [:= :entity.db_id :database.id])
(:collection joins) (conj :collection [:= :entity.collection_id :collection.id])
@@ -457,13 +496,14 @@
(:document joins) (conj :document [:= :entity.document_id :document.id])
(:table joins) (conj (warehouse-schema-overlay/table-query {:alias :table}) [:= :entity.table_id :table.id])))
-(defn- dependency-item-select
+(mu/defn- dependency-item-select :- ::h2x/honeysql-clause-opts
"The per-entity-type SELECT that [[dependency-item-ids]] and [[dependency-item-count]] union together, matching
`query-type` (`:unreferenced` or `:breaking`) items of `entity-type`, restricted to `card-types`, `search-text`,
and `include-personal-collections?`, visible to the user described by `user-id`, `is-superuser?`, and
`is-data-analyst?`, with `sort-column`'s expression selected as `:sort_key`. Throws when `user-id` is missing,
since the visibility restriction must always be applied."
- [{:keys [query-type entity-type sort-column user-id is-superuser? is-data-analyst?] :as params}]
+ [{:keys [query-type entity-type sort-column user-id is-superuser? is-data-analyst?] :as params}
+ :- DependencyItemSelectParams]
(when-not user-id
(throw (ex-info "dependency-item-select requires a user-id so the visibility restriction is always applied"
{:query-type query-type :entity-type entity-type})))
@@ -489,22 +529,6 @@
:left-join (build-left-joins join all-joins)
:where (into [:and join-filter item-visible-expr] (keep identity) exprs)}))
-(def ^:private DependencyItemParams
- "Params shared by [[dependency-item-ids]] and [[dependency-item-count]]."
- [:map {:closed true}
- [:query-type [:enum :unreferenced :breaking]]
- [:entity-types [:sequential ::deps.dependency-types/dependency-types]]
- [:card-types {:optional true} [:maybe [:sequential [:or :keyword :string]]]]
- [:search-text {:optional true} [:maybe :string]]
- [:include-personal-collections? :boolean]
- [:sort-column {:optional true} [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]]
- [:sort-direction [:enum :asc :desc]]
- [:offset ms/IntGreaterThanOrEqualToZero]
- [:limit ms/PositiveInt]
- [:user-id ::lib.schema.id/user]
- [:is-superuser? {:optional true} [:maybe :boolean]]
- [:is-data-analyst? {:optional true} [:maybe :boolean]]])
-
(mu/defn dependency-item-ids
"A page of `[entity-type entity-id]` pairs for `query-type` (`:unreferenced` or `:breaking`), restricted to
`entity-types`, `card-types` (applied only to `:card` entities), `search-text` (nil for none), and
@@ -538,7 +562,7 @@
[{:keys [source-entity-type source-entity-id dependent-types dependent-card-types
user-id is-superuser? is-data-analyst?]}
:- [:map {:closed true}
- [:source-entity-type EntityType]
+ [:source-entity-type ::dependencies.schema/entity-type]
[:source-entity-id ms/PositiveInt]
[:dependent-types {:optional true} [:maybe [:sequential :string]]]
[:dependent-card-types {:optional true} [:maybe [:sequential :string]]]
@@ -550,8 +574,8 @@
:from [[:analysis_finding_error :afe]]
:join [[:analysis_finding :af]
[:and
- [:= :af.analyzed_entity_type :afe.analyzed_entity_type]
- [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]
+ [:= :af.entity_type :afe.analyzed_entity_type]
+ [:= :af.entity_id :afe.analyzed_entity_id]]]
:where (cond-> [:and
[:= :afe.source_entity_type (name source-entity-type)]
[:= :afe.source_entity_id source-entity-id]
@@ -571,29 +595,30 @@
;;; ---------------------------------------------------- Entities ----------------------------------------------------
-(mu/defn instances
- "The instances of the entity type `entity-type` with `ids`."
+(mu/defn entities
+ "The entities of the entity type `entity-type` with `ids`."
[entity-type :- ::deps.dependency-types/dependency-types
ids :- [:set ::deps.dependency-types/entity-id]]
(t2/select (deps.dependency-types/dependency-type->model entity-type) :id [:in ids]))
-(defn- entity-source
+(mu/defn- entity-source :- [:map {:closed true}
+ [:from {:optional true} [:sequential HoneySQLForm]]]
"What a read of `entity-type` selects from: the overlay for a Table, else the model's own table."
- [entity-type]
+ [entity-type :- ::deps.dependency-types/dependency-types]
(if (= entity-type :table)
{:from [(warehouse-schema-overlay/table-query)]}
{}))
-(mu/defn instances-with-columns
- "The `columns` of the instances of the entity type `entity-type` with `ids`."
+(mu/defn entities-with-columns
+ "The `columns` of the entities of the entity type `entity-type` with `ids`."
[entity-type :- ::deps.dependency-types/dependency-types
columns :- [:sequential :keyword]
ids :- [:sequential ::deps.dependency-types/entity-id]]
(t2/select (into [(deps.dependency-types/dependency-type->model entity-type)] columns)
:id [:in ids] (entity-source entity-type)))
-(mu/defn instance-with-columns
- "The `columns` of the instance of the entity type `entity-type` with `id`, or nil."
+(mu/defn entity-with-columns
+ "The `columns` of the entity of the entity type `entity-type` with `id`, or nil."
[entity-type :- ::deps.dependency-types/dependency-types
columns :- [:sequential :keyword]
id :- ::deps.dependency-types/entity-id]
@@ -645,16 +670,16 @@
[transform-ids :- [:sequential ::lib.schema.id/transform]]
(t2/select [:model/Transform :id :source] :id [:in transform-ids]))
-(mu/defn transform-ids-of-source-database
+(mu/defn source-database-transform-ids :- [:set ::lib.schema.id/transform]
"The IDs of the Transforms reading from the Database with `db-id`."
[db-id :- ::lib.schema.id/database]
- (t2/select-pks-set :model/Transform :source_database_id db-id))
+ (or (t2/select-pks-set :model/Transform :source_database_id db-id) #{}))
;;; -------------------------------------------------- Dependencies --------------------------------------------------
(mu/defn dependencies-from
"The `:id`, `:to_entity_type`, and `:to_entity_id` of the Dependencies of the entity `entity-type` `entity-id`."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-id :- ms/PositiveInt]
(t2/select [:model/Dependency :id :to_entity_type :to_entity_id]
:from_entity_type entity-type
@@ -662,9 +687,9 @@
(mu/defn dependency-exists?
"Whether the entity `from-type` `from-id` depends on the entity `to-type` `to-id`."
- [from-type :- EntityType
+ [from-type :- ::dependencies.schema/entity-type
from-id :- ms/PositiveInt
- to-type :- EntityType
+ to-type :- ::dependencies.schema/entity-type
to-id :- ms/PositiveInt]
(t2/exists? :model/Dependency
:from_entity_type from-type :from_entity_id from-id
@@ -683,11 +708,11 @@
(mu/defn retarget-dependency!
"Point the Dependency of the entity `from-type` `from-id` on the entity `old-to-type` `old-to-id` at the entity
`new-to-type` `new-to-id`, returning the number updated."
- [from-type :- EntityType
+ [from-type :- ::dependencies.schema/entity-type
from-id :- ms/PositiveInt
- old-to-type :- EntityType
+ old-to-type :- ::dependencies.schema/entity-type
old-to-id :- ms/PositiveInt
- new-to-type :- EntityType
+ new-to-type :- ::dependencies.schema/entity-type
new-to-id :- ms/PositiveInt]
(t2/update! :model/Dependency
{:from_entity_type from-type :from_entity_id from-id
@@ -702,9 +727,9 @@
(mu/defn delete-dependency!
"Delete the Dependency of the entity `from-type` `from-id` on the entity `to-type` `to-id`, returning the number
deleted."
- [from-type :- EntityType
+ [from-type :- ::dependencies.schema/entity-type
from-id :- ms/PositiveInt
- to-type :- EntityType
+ to-type :- ::dependencies.schema/entity-type
to-id :- ms/PositiveInt]
(t2/delete! :model/Dependency
:from_entity_type from-type :from_entity_id from-id
@@ -712,7 +737,7 @@
(mu/defn delete-dependencies-from!
"Delete the Dependencies of the entities `entity-type` `entity-ids`, returning the number deleted."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-ids :- [:sequential ms/PositiveInt]]
(t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id [:in entity-ids]))
@@ -735,250 +760,360 @@
:to_entity_type :transform
:to_entity_id transform-id))
-;;; ------------------------------------------------ Dependency status ------------------------------------------------
-
-(def ^:private entity-database-queries
- "For each entity type that can hold a native query, a query of the IDs of its entities reading from `db-id`."
- {:card (fn [db-id] ^:allow-subquery {:select [:id]
- :from [(t2/table-name :model/Card)]
- :where [:= :database_id db-id]})
- :transform (fn [db-id] ^:allow-subquery {:select [:id]
- :from [(t2/table-name :model/Transform)]
- :where [:= :source_database_id db-id]})
- :action (fn [db-id] ^:allow-subquery {:select [:action_id]
- :from [(t2/table-name :model/QueryAction)]
- :where [:= :database_id db-id]})})
-
-(mu/defn unresolved-table-entity-ids :- [:map-of ::deps.dependency-types/dependency-types [:set ms/PositiveInt]]
- "The IDs, by entity type, of the entities reading from the Database with `db-id` that name tables matching nothing."
- [db-id :- ::lib.schema.id/database]
- (into {}
- (keep (fn [[entity-type entity-ids-query]]
- (when-let [ids (not-empty (t2/select-fn-set :entity_id :model/DependencyStatus
- :entity_type entity-type
- :unresolved_tables true
- :entity_id [:in (entity-ids-query db-id)]))]
- [entity-type ids])))
- entity-database-queries))
-
-(mu/defn dependency-status
- "The DependencyStatus of the entity `entity-type` `entity-id`, or nil."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt]
- (t2/select-one :model/DependencyStatus :entity_type entity-type :entity_id entity-id))
+;;; -------------------------------------------- Statuses, shared by the jobs --------------------------------------------
+
+(def ^:private StatusModel
+ "A job's status model."
+ [:enum :model/DependencyStatus :model/ResultMetadataStatus :model/AnalysisFindingStatus])
+
+(def ^:private one-more-write
+ "The `stale` count of a status after one more write changed what its job reads, as Honey SQL."
+ [:+ :stale [:inline 1]])
+
+(def ^:private fresh-attempts
+ "The columns of a status a write resets, so that the job tries an entity it gave up on again."
+ {:fail_count 0 :terminal false})
+
+(mu/defn- mark-stale!
+ "Count one more write in the statuses in the status `model` of the entities `type->ids`, with fresh attempts, so the
+ job processes them until it clears a count no write raised meanwhile; an entity without a status is processed
+ anyway."
+ [model :- StatusModel
+ type->ids :- ::dependencies.schema/type->ids]
+ (doseq [[entity-type ids] type->ids
+ batch (partition-all 1000 ids)]
+ (t2/update! model :entity_type entity-type :entity_id [:in batch] (assoc fresh-attempts :stale one-more-write))))
+
+(mu/defn- mark-dependents-stale!
+ "Count one more write in the statuses in the status `model` of the entities directly depending on the entities
+ `entity-type` `entity-ids`, in one UPDATE per 1,000 of them."
+ [model :- StatusModel
+ entity-type :- ::dependencies.schema/entity-type
+ entity-ids :- [:set ms/PositiveInt]]
+ (let [table (t2/table-name model)]
+ (doseq [batch (partition-all 1000 entity-ids)]
+ (t2/query {:update table
+ :set (assoc fresh-attempts :stale one-more-write)
+ :where [:exists ^:allow-subquery
+ {:select [1]
+ :from [[(t2/table-name :model/Dependency) :d]]
+ :where [:and
+ [:= :d.to_entity_type (name entity-type)]
+ [:in :d.to_entity_id batch]
+ [:= :d.from_entity_type (u/qualified-key table :entity_type)]
+ [:= :d.from_entity_id (u/qualified-key table :entity_id)]]}]}))))
+
+(mu/defn- clear-stale! :- :boolean
+ "Record that a job processed the entity `entity-type` `entity-id` with its logic of `current-version`, clearing the
+ `stale` count of its status in the status `model` when it is still the `stale` the job picked it with, so a write
+ while it was processed has the job process it again, returning whether it cleared it."
+ [model :- StatusModel
+ entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ stale :- ms/IntGreaterThanOrEqualToZero
+ current-version :- ms/PositiveInt]
+ (pos? (t2/update! model :entity_type entity-type :entity_id entity-id :stale stale
+ {:stale 0 :fail_count 0 :analysis_version current-version})))
+
+(mu/defn- record-failure! :- :boolean
+ "Record that a job failed to process the entity `entity-type` `entity-id`, giving up on it until a write changes it
+ once it failed `max-failures` times in a row, returning whether it gave up."
+ [model :- StatusModel
+ entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ max-failures :- ms/PositiveInt]
+ (t2/update! model :entity_type entity-type :entity_id entity-id {:fail_count [:+ :fail_count [:inline 1]]})
+ (pos? (t2/update! model :entity_type entity-type :entity_id entity-id :fail_count [:>= max-failures]
+ {:terminal true})))
+
+(mu/defn mark-dependency-statuses-stale!
+ "Mark the DependencyStatuses of the entities `type->ids` stale, so the dependency job computes their dependencies
+ again."
+ [type->ids :- ::dependencies.schema/type->ids]
+ (mark-stale! :model/DependencyStatus type->ids))
+
+(mu/defn mark-result-metadata-statuses-stale!
+ "Mark the ResultMetadataStatuses of the entities `type->ids` stale, so the result metadata job re-derives them."
+ [type->ids :- ::dependencies.schema/type->ids]
+ (mark-stale! :model/ResultMetadataStatus type->ids))
+
+(mu/defn mark-analysis-finding-statuses-stale!
+ "Mark the AnalysisFindingStatuses of the entities `type->ids` stale, so the analysis job analyzes them again."
+ [type->ids :- ::dependencies.schema/type->ids]
+ (mark-stale! :model/AnalysisFindingStatus type->ids))
+
+(mu/defn mark-dependents-result-metadata-statuses-stale!
+ "Mark the ResultMetadataStatuses of the entities directly depending on the entities `entity-type` `entity-ids` stale."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-ids :- [:set ms/PositiveInt]]
+ (mark-dependents-stale! :model/ResultMetadataStatus entity-type entity-ids))
+
+(mu/defn mark-dependents-analysis-finding-statuses-stale!
+ "Mark the AnalysisFindingStatuses of the entities directly depending on the entities `entity-type` `entity-ids`
+ stale."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-ids :- [:set ms/PositiveInt]]
+ (mark-dependents-stale! :model/AnalysisFindingStatus entity-type entity-ids))
+
+(mu/defn- current-dependencies-clause
+ "Whether the dependency status of the entity of `entity-type` with `id-field` is current for the dependency logic of
+ `dependency-version`, as Honey SQL."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ id-field :- :keyword
+ dependency-version :- ms/PositiveInt]
+ [:exists ^:allow-subquery {:select [1]
+ :from [[:dependency_status :ds]]
+ :where [:and
+ [:= :ds.entity_type (name entity-type)]
+ [:= :ds.entity_id id-field]
+ [:= :ds.stale 0]
+ [:= :ds.analysis_version dependency-version]]}])
+
+(mu/defn- pending-statuses :- [:sequential ::dependencies.schema/pending-status]
+ "Up to `batch-size` entities of `entity-type` without a status in `status-model`, or whose status the job has not
+ given up on and that counts a write or records a version of the job's logic below `current-version`, those last,
+ with their `stale` counts, nil without a status; `user-mbql-only?` narrows cards to MBQL ones that Metabase did not create itself, and `dependency-version`
+ to the entities whose dependencies are current."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ status-model :- StatusModel
+ batch-size :- ms/PositiveInt
+ current-version :- ms/PositiveInt
+ {:keys [user-mbql-only? dependency-version]} :- [:map {:closed true}
+ [:user-mbql-only? {:optional true} :boolean]
+ [:dependency-version {:optional true} ms/PositiveInt]]]
+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))
+ id-field (u/qualified-key table-name :id)
+ outdated [:< :status.analysis_version current-version]]
+ (mapv #(select-keys % [:entity_id :stale])
+ (t2/query
+ {:select [[id-field :entity_id] [:status.stale :stale]]
+ :from [table-name]
+ :left-join [[(t2/table-name status-model) :status] [:and
+ [:= :status.entity_type (name entity-type)]
+ [:= :status.entity_id id-field]]]
+ :where (cond-> [:and [:or [:= :status.id nil]
+ [:and [:= :status.terminal false] [:or [:> :status.stale 0] outdated]]]]
+ user-mbql-only? (conj [:or [:= :report_card.query_type nil]
+ [:not= :report_card.query_type "native"]]
+ (mi/exclude-internal-content-hsql :model/Card
+ :table-alias :report_card))
+ dependency-version (conj (current-dependencies-clause
+ entity-type id-field dependency-version)))
+ :order-by [[[:case outdated [:inline 1] :else [:inline 0]]]]
+ :limit batch-size}))))
+
+(mu/defn- take-pending-statuses! :- [:sequential ::dependencies.schema/pending-status]
+ "The [[pending-statuses]] a job is to process, each with the `stale` count it clears afterwards, inserting a status
+ counting one write for each entity without one before the job reads it, so a write while it is processed counts."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ status-model :- StatusModel
+ batch-size :- ms/PositiveInt
+ current-version :- ms/PositiveInt
+ opts :- [:map {:closed true}
+ [:user-mbql-only? {:optional true} :boolean]
+ [:dependency-version {:optional true} ms/PositiveInt]]]
+ (let [statuses (pending-statuses entity-type status-model batch-size current-version opts)
+ missing (remove :stale statuses)]
+ (when (seq missing)
+ (t2/insert! status-model (for [{:keys [entity_id]} missing]
+ {:entity_type entity-type
+ :entity_id entity_id
+ :stale 1
+ :analysis_version 0})))
+ (mapv #(update % :stale (fnil identity 1)) statuses)))
+
+(mu/defn- deleted-entity-ids-with-status :- [:sequential ms/PositiveInt]
+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have a status in `status-model`."
+ [status-model :- StatusModel
+ entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]
+ (mapv :entity_id (t2/query {:select [:status.entity_id]
+ :from [[(t2/table-name status-model) :status]]
+ :where [:and
+ [:= :status.entity_type (name entity-type)]
+ [:not [:exists ^:allow-subquery {:select [1]
+ :from [[table-name :entity]]
+ :where [:= :entity.id :status.entity_id]}]]]
+ :limit batch-size}))))
+
+;;; ---------------------------------------------- Dependency job ----------------------------------------------
+
+(mu/defn dependency-deleted-entity-ids :- [:sequential ms/PositiveInt]
+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have a DependencyStatus."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (deleted-entity-ids-with-status :model/DependencyStatus entity-type batch-size))
(mu/defn delete-dependency-statuses!
"Delete the DependencyStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-ids :- [:sequential ms/PositiveInt]]
(t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id [:in entity-ids]))
-(mu/defn deleted-entity-ids :- [:sequential ms/PositiveInt]
- "Up to `batch-size` IDs of deleted entities of the type `entity-type` that still have a DependencyStatus."
- [entity-type :- ::deps.dependency-types/dependency-types
- batch-size :- ms/PositiveInt]
- (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]
- (vec (t2/select-fn-vec :entity_id :model/DependencyStatus
- {:where [:and
- [:= :dependency_status.entity_type (name entity-type)]
- [:not [:exists ^:allow-subquery
- {:select [1]
- :from [[table-name :entity]]
- :where [:= :entity.id :dependency_status.entity_id]}]]]
- :limit batch-size}))))
-
-(mu/defn mark-dependency-status-stale!
- "Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it
- doesn't exist. Resets retry state so previously-failed entities get a fresh chance."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt]
- (mdb/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id entity-id}
- (fn [existing]
- (if existing
- {:stale true :fail_count 0 :next_retry_at nil :terminal false}
- {:stale true :dependency_analysis_version 0}))))
-
-(mu/defn upsert-dependency-status!
- "Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to
- `current-version`, `unresolved_tables` to `unresolved-tables?`, and clearing any failure state."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt
- current-version :- ms/PositiveInt
- unresolved-tables? :- :boolean]
- (mdb/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id entity-id}
- (fn [_existing]
- {:dependency_analysis_version current-version
- :stale false
- :fail_count 0
- :next_retry_at nil
- :unresolved_tables unresolved-tables?})))
-
-(mu/defn record-dependency-status-failure!
- "Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives the
- existing DependencyStatus row (or nil if none exists) and must return the columns to set."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt
- update-fn :- fn?]
- (mdb/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id entity-id}
- update-fn))
-
-(mu/defn pending-retry-exists?
- "Whether a non-terminal DependencyStatus is waiting for a retry."
- []
- (t2/exists? :model/DependencyStatus :terminal false :next_retry_at [:not= nil]))
+(mu/defn record-dependency-failure! :- :boolean
+ "Record that the dependency job failed to compute the dependencies of the entity `entity-type` `entity-id`, giving up on it until a write changes it once it failed
+ `max-failures` times in a row, returning whether it gave up."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ max-failures :- ms/PositiveInt]
+ (record-failure! :model/DependencyStatus entity-type entity-id max-failures))
+
+(mu/defn clear-dependency-status! :- :boolean
+ "Record that the dependency job computed the dependencies of the entity `entity-type` `entity-id` with the analysis
+ of `current-version`, clearing its count of `stale` writes unless a write raised it, returning whether it cleared it."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ stale :- ms/IntGreaterThanOrEqualToZero
+ current-version :- ms/PositiveInt]
+ (clear-stale! :model/DependencyStatus entity-type entity-id stale current-version))
+
+(mu/defn dependency-pending? :- :boolean
+ "Whether an entity of the type `entity-type` has no DependencyStatus, or one that counts a write or records a version
+ of the dependency logic below `current-version`."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ current-version :- ms/PositiveInt]
+ (boolean (seq (pending-statuses entity-type :model/DependencyStatus 1 current-version {}))))
-(mu/defn instances-for-dependency-calculation
- "Up to `batch-size` instances of the entity type `entity-type` without a DependencyStatus, or whose status is
- stale or below `current-version`, is not terminal, and whose retry time has passed at `now`; stale ones first."
+(mu/defn take-pending-dependency-statuses! :- [:sequential ::dependencies.schema/pending-status]
+ "Up to `batch-size` entities of the type `entity-type` without a DependencyStatus, or whose status counts a write or
+ records a version of the dependency logic below `current-version`, outdated ones last, with the `stale` count the job
+ clears afterwards."
[entity-type :- ::deps.dependency-types/dependency-types
batch-size :- ms/PositiveInt
- current-version :- ms/PositiveInt
- now :- ms/TemporalInstant]
- (let [model (deps.dependency-types/dependency-type->model entity-type)
- table-name (t2/table-name model)
- id-field (keyword (name table-name) "id")
- table-wildcard (keyword (name table-name) "*")]
- (t2/select model
- {:select [table-wildcard]
- :from table-name
- :left-join [:dependency_status [:and
- [:= :dependency_status.entity_id id-field]
- [:= :dependency_status.entity_type (name entity-type)]]]
- :where [:or
- [:= :dependency_status.entity_id nil]
- [:and
- [:or
- [:= :dependency_status.stale true]
- [:< :dependency_status.dependency_analysis_version current-version]]
- [:= :dependency_status.terminal false]
- [:or
- [:is :dependency_status.next_retry_at nil]
- [:<= :dependency_status.next_retry_at now]]]]
- :order-by [[[:case
- [:or [:= :dependency_status.entity_id nil] [:= :dependency_status.stale true]] [:inline 0]
- :else [:inline 1]]]]
- :limit batch-size})))
-
-;;; ------------------------------------------------ Analysis findings ------------------------------------------------
-
-(mu/defn finding-id
- "The ID of the AnalysisFinding of the entity `entity-type` `entity-id`, or nil."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt]
- (t2/select-one-fn :id [:model/AnalysisFinding :id]
- :analyzed_entity_type entity-type
- :analyzed_entity_id entity-id))
-
-(mu/defn insert-finding!
- "Insert the AnalysisFinding `row`, returning the number inserted."
- [row :- [:map {:closed true}
- [:id {:optional true} ms/PositiveInt]
- [:analyzed_at {:optional true} [:maybe ms/TemporalInstant]]
- [:analysis_version {:optional true} ms/PositiveInt]
- [:result {:optional true} :boolean]
- [:stale {:optional true} :boolean]
- [:analyzed_entity_type {:optional true} [:maybe [:or :keyword :string]]]
- [:analyzed_entity_id {:optional true} [:maybe ms/PositiveInt]]]]
- (t2/insert! :model/AnalysisFinding row))
-
-(mu/defn update-finding!
- "Apply `changes` to the AnalysisFinding with `finding-id`, returning the number updated."
- [finding-id :- ms/PositiveInt
- changes :- (mut/select-keys ::dependencies.schema/analysis-finding.update [:analyzed_at :analysis_version :result :stale])]
- (t2/update! :model/AnalysisFinding finding-id changes))
-
-(mu/defn mark-findings-stale!
- "Mark the AnalysisFindings of the entities `entity-type` `entity-ids` as stale, returning the number updated."
- [entity-type :- EntityType
- entity-ids :- [:sequential ms/PositiveInt]]
- (t2/update! :model/AnalysisFinding
- :analyzed_entity_type entity-type
- :analyzed_entity_id [:in entity-ids]
- {:stale true}))
+ current-version :- ms/PositiveInt]
+ (take-pending-statuses! entity-type :model/DependencyStatus batch-size current-version {}))
-(mu/defn stale-finding-exists?
- "Whether a stale AnalysisFinding exists."
- []
- (t2/exists? :model/AnalysisFinding :stale true))
+;;; ------------------------------------------- Result metadata job -------------------------------------------
-(mu/defn stale-finding-count
- "The number of stale AnalysisFindings."
- []
- (t2/count :model/AnalysisFinding :stale true))
+(mu/defn take-pending-result-metadata-statuses! :- [:sequential ::dependencies.schema/pending-status]
+ "Up to `batch-size` MBQL cards Metabase did not create itself without a ResultMetadataStatus, or whose status counts a
+ write or records a version of the logic below `current-version`, and whose dependencies the dependency logic of
+ `dependency-version` computed, outdated ones last, with the `stale` count the job clears afterwards."
+ [batch-size :- ms/PositiveInt
+ current-version :- ms/PositiveInt
+ dependency-version :- ms/PositiveInt]
+ (take-pending-statuses! :card :model/ResultMetadataStatus batch-size current-version
+ {:user-mbql-only? true
+ :dependency-version dependency-version}))
+
+(mu/defn record-result-metadata-failure! :- :boolean
+ "Record that the result metadata job failed to re-derive the Card with `card-id`, giving up on it until a write
+ changes it once it failed `max-failures` times in a row, returning whether it gave up."
+ [card-id :- ::lib.schema.id/card
+ max-failures :- ms/PositiveInt]
+ (record-failure! :model/ResultMetadataStatus :card card-id max-failures))
+
+(mu/defn clear-result-metadata-status! :- :boolean
+ "Record that the result metadata job re-derived the Card with `card-id` with the logic of `current-version`, clearing
+ its count of `stale` writes unless a write raised it, returning whether it cleared it."
+ [card-id :- ::lib.schema.id/card
+ stale :- ms/IntGreaterThanOrEqualToZero
+ current-version :- ms/PositiveInt]
+ (clear-stale! :model/ResultMetadataStatus :card card-id stale current-version))
-(mu/defn instances-for-analysis
- "Up to `batch-size` instances of the entity type `entity-type` whose AnalysisFinding is stale, missing, or below
- `current-version`; stale ones first, then the longest unanalyzed."
- [entity-type :- ::deps.dependency-types/dependency-types
- batch-size :- ms/PositiveInt
+(mu/defn result-metadata-deleted-card-ids :- [:sequential ::lib.schema.id/card]
+ "Up to `batch-size` IDs of deleted Cards that still have a ResultMetadataStatus."
+ [batch-size :- ms/PositiveInt]
+ (deleted-entity-ids-with-status :model/ResultMetadataStatus :card batch-size))
+
+(mu/defn delete-result-metadata-statuses!
+ "Delete the ResultMetadataStatuses of the Cards with `card-ids`, returning the number deleted."
+ [card-ids :- [:sequential ::lib.schema.id/card]]
+ (t2/delete! :model/ResultMetadataStatus :entity_type :card :entity_id [:in card-ids]))
+
+(mu/defn remapped-table-ids :- [:set ::lib.schema.id/table]
+ "The IDs of the Tables of the Fields remapped to one of the Fields with `field-ids` as their human-readable field."
+ [field-ids :- [:set ::lib.schema.id/field]]
+ (or (t2/select-fn-set :table_id [:model/Field :metabase_field.table_id]
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :join [:dimension [:= :dimension.field_id :metabase_field.id]]
+ :where [:in :dimension.human_readable_field_id field-ids]})
+ #{}))
+
+(mu/defn internal-remap? :- :boolean
+ "Whether the Field with `field-id` has an internal remap, which shows its FieldValues' human-readable values."
+ [field-id :- ::lib.schema.id/field]
+ (t2/exists? :model/Dimension :field_id field-id :type "internal"))
+
+(mu/defn native-card-ids :- [:set ::lib.schema.id/card]
+ "The IDs of the native Cards among `card-ids`."
+ [card-ids :- [:set ::lib.schema.id/card]]
+ (or (t2/select-pks-set :model/Card :id [:in card-ids] :query_type :native) #{}))
+
+(mu/defn card-state :- [:maybe ::dependencies.schema/card-state]
+ "The `updated_at` and `result_metadata` of the Card with `card-id`, locking its row until the transaction ends when
+ `lock?`, or nil."
+ [card-id :- ::lib.schema.id/card
+ lock? :- :boolean]
+ (some-> (t2/select-one [:model/Card :updated_at :result_metadata]
+ (cond-> {:where [:= :id card-id]} lock? (assoc :for :update)))
+ (select-keys [:updated_at :result_metadata])))
+
+(mu/defn set-card-result-metadata!
+ "Set the result metadata of the Card with `card-id` to `result-metadata` through the Card model, without touching
+ `updated_at`."
+ [card-id :- ::lib.schema.id/card
+ result-metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]]
+ (t2/update! :model/Card card-id {:result_metadata result-metadata
+ :updated_at :updated_at}))
+
+;;; ------------------------------------------- Analysis finding job -------------------------------------------
+
+(mu/defn upsert-finding!
+ "Insert or update the AnalysisFinding of the entity `entity-type` `entity-id` with `result`, updating the row a
+ concurrent analysis inserted meanwhile."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ result :- :boolean]
+ (mdb/update-or-insert! :model/AnalysisFinding {:entity_type entity-type :entity_id entity-id}
+ (constantly {:result result})))
+
+(mu/defn take-pending-analysis-finding-statuses! :- [:sequential ::dependencies.schema/pending-status]
+ "Up to `batch-size` entities of the type `entity-type` without an AnalysisFindingStatus, or whose status counts a
+ write or records an analysis below `current-version`, and whose dependencies the analysis of `dependency-version`
+ computed, outdated ones last, with the `stale` count the job clears afterwards."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt
+ current-version :- ms/PositiveInt
+ dependency-version :- ms/PositiveInt]
+ (take-pending-statuses! entity-type :model/AnalysisFindingStatus batch-size current-version
+ {:dependency-version dependency-version}))
+
+(mu/defn record-analysis-finding-failure! :- :boolean
+ "Record that the analysis job failed to analyze the entity `entity-type` `entity-id`, giving up on it until a write changes it once it failed
+ `max-failures` times in a row, returning whether it gave up."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ max-failures :- ms/PositiveInt]
+ (record-failure! :model/AnalysisFindingStatus entity-type entity-id max-failures))
+
+(mu/defn clear-analysis-finding-status! :- :boolean
+ "Record that the analysis job analyzed the entity `entity-type` `entity-id` with the analysis of `current-version`,
+ clearing its count of `stale` writes unless a write raised it, returning whether it cleared it."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ stale :- ms/IntGreaterThanOrEqualToZero
current-version :- ms/PositiveInt]
- (let [model (deps.dependency-types/dependency-type->model entity-type)
- table-name (t2/table-name model)
- id-field (keyword (name table-name) "id")
- table-wildcard (keyword (name table-name) "*")]
- (t2/select model
- {:select [table-wildcard]
- :from table-name
- :left-join [:analysis_finding [:and
- [:= :analysis_finding.analyzed_entity_id id-field]
- [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]
- :where [:or
- [:= :analysis_finding.stale true]
- [:<
- [:coalesce :analysis_finding.analysis_version 0]
- current-version]]
- :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]
- [:analysis_finding.analyzed_at :asc]]
- :limit batch-size})))
-
-(mu/defn table-ids-with-outdated-findings
- "The IDs of the Tables of the Database with `db-id` whose dependents were analyzed before the Table, one of its
- Fields, or one of its Fields' user settings last changed."
- [db-id :- ::lib.schema.id/database]
- (t2/select-fn-set :table_id :model/AnalysisFinding
- {:select [:field_updates/table_id]
- :from [[^:allow-subquery {:select [[:table/id :table_id]
- [:table/updated_at :last_table_update]
- [[:max [:case
- [:>= :field/updated_at
- [:coalesce :field_settings/updated_at :field/updated_at]]
- :field/updated_at
- :else :field_settings/updated_at]]
- :last_field_update]]
- :from [(warehouse-schema-overlay/table-query {:alias :table})]
- :left-join [(warehouse-schema-overlay/field-query {:alias :field})
- [:= :field/table_id :table/id]
- [(t2/table-name :model/FieldUserSettings) :field_settings]
- [:= :field_settings/field_id :field/id]]
- :where [:= :table/db_id db-id]
- :group-by [:table/id
- :table/updated_at]}
- :field_updates]]
- :inner-join [[(t2/table-name :model/Dependency) :dep]
- [:and
- [:= :dep/to_entity_type "table"]
- [:= :field_updates/table_id :dep/to_entity_id]]
- [(t2/table-name :model/AnalysisFinding) :finding]
- [:and
- [:= :finding/analyzed_entity_type :dep/from_entity_type]
- [:= :finding/analyzed_entity_id :dep/from_entity_id]]]
- :where [:and
- [:!= :finding/analyzed_entity_id nil]
- [:or
- [:< :finding/analyzed_at :field_updates/last_table_update]
- [:< :finding/analyzed_at :field_updates/last_field_update]]]}))
+ (clear-stale! :model/AnalysisFindingStatus entity-type entity-id stale current-version))
+
+(mu/defn analysis-finding-deleted-entity-ids :- [:sequential ms/PositiveInt]
+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have an AnalysisFindingStatus."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (deleted-entity-ids-with-status :model/AnalysisFindingStatus entity-type batch-size))
+
+(mu/defn delete-analysis-finding-statuses!
+ "Delete the AnalysisFindingStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-ids :- [:sequential ms/PositiveInt]]
+ (t2/delete! :model/AnalysisFindingStatus :entity_type entity-type :entity_id [:in entity-ids]))
;;; --------------------------------------------- Analysis finding errors ---------------------------------------------
(mu/defn finding-errors-for-entity
"The AnalysisFindingErrors of the entity `entity-type` `entity-id`."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-id :- ms/PositiveInt]
(t2/select :model/AnalysisFindingError :analyzed_entity_type entity-type :analyzed_entity_id entity-id))
@@ -995,6 +1130,6 @@
(mu/defn delete-finding-errors-for-entity!
"Delete the AnalysisFindingErrors of the entity `entity-type` `entity-id`, returning the number deleted."
- [entity-type :- EntityType
+ [entity-type :- ::dependencies.schema/entity-type
entity-id :- ms/PositiveInt]
(t2/delete! :model/AnalysisFindingError :analyzed_entity_type entity-type :analyzed_entity_id entity-id))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj b/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj
index 1a68b578756..d0129ed652d 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj
@@ -26,7 +26,7 @@
(-> model->dependency-type vals set))
(def backfillable-dependency-types
- "Dependency types that the backfill job computes dependencies for. Excludes `:table`: tables aren't
+ "Dependency types that the dependency job computes dependencies for. Excludes `:table`: tables aren't
backfilled directly; links involving tables are found via analysis of the other side of the relation."
(disj dependency-types :table))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
deleted file mode 100644
index 990c09a3d52..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ /dev/null
@@ -1,178 +0,0 @@
-(ns metabase-enterprise.dependencies.findings
- (:require
- [metabase-enterprise.dependencies.analysis :as deps.analysis]
- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
- [metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]
- [toucan2.core :as t2]))
-
-(set! *warn-on-reflection* true)
-
-(defmulti ^:private instance-db-id
- "Gets the database id for a toucan instance"
- {:arglists '([toucan-instance])}
- t2/model)
-
-(defmethod instance-db-id :default
- [toucan-instance]
- (:database_id toucan-instance))
-
-(defmethod instance-db-id :model/Transform
- [toucan-instance]
- (or (some-> toucan-instance :source :query lib/database-id)
- ;; non-query transforms (e.g. Python) have no query to read the database from
- (:source_database_id toucan-instance)))
-
-(defmethod instance-db-id :model/Segment
- [toucan-instance]
- (some-> toucan-instance :definition lib/database-id))
-
-(defmulti ^:private pre-analysis-errors
- "Return a (possibly empty) sequence of errors detected without running the full analyzer. When
- non-empty, the entity is recorded broken with these errors and the regular analysis is skipped.
- Use this for structural failures e.g. an orphan Transform whose source database has been deleted."
- {:arglists '([toucan-instance])}
- t2/model)
-
-(defmethod pre-analysis-errors :default [_] nil)
-
-(defmethod pre-analysis-errors :model/Transform
- [{db-id :source_database_id id :id}]
- (when (nil? db-id)
- [(-> (lib/validation-exception-error "Source database for this transform has been deleted.")
- (assoc :source-entity-type :transform :source-entity-id id))]))
-
-(defn upsert-analysis!
- "Given a Toucan entity, run its analysis and write the results into `:model/AnalysisFinding`.
-
- If any row exists already, it is replaced. If it does not exist, it is created."
- [toucan-instance]
- (when-not (lib-be/metadata-provider-cache)
- (throw (ex-info "FIXME: deps.findings/upsert-analysis! ran without reusing `MetadataProvider`s"
- {:instance toucan-instance})))
- (let [model (t2/model toucan-instance)
- entity-type (deps.dependency-types/model->dependency-type model)
- instance-id (:id toucan-instance)]
- (if-let [errors (seq (pre-analysis-errors toucan-instance))]
- (deps.analysis-finding/upsert-analysis! entity-type instance-id false errors)
- (if-let [db-id (instance-db-id toucan-instance)]
- (let [mp (lib-be/application-database-metadata-provider db-id)
- results (try (deps.analysis/check-entity mp entity-type instance-id)
- (catch Exception e
- (log/errorf "Error analyzing entity: %s" (ex-message e))
- [(lib/validation-exception-error (.getMessage e))]))
- success (empty? results)]
- (deps.analysis-finding/upsert-analysis! entity-type instance-id success results))
- ;; No resolvable database, and no pre-analysis error explaining it: record a terminal error so
- ;; the entity gets a finding (clearing its stale flag) instead of no-oping forever and being
- ;; re-selected on every run. It re-checks normally if its database later becomes resolvable.
- (deps.analysis-finding/upsert-analysis!
- entity-type instance-id false
- [(lib/validation-exception-error "Could not resolve a database for this entity.")])))))
-
-(defn analyze-instances!
- "Given a series of toucan entities, upsert analyses for all of them and catch errors."
- [instances]
- (doseq [instance instances]
- (try (upsert-analysis! instance)
- (catch Exception e
- (log/errorf "Analyzing entity %s %s failed: %s"
- (t2/model instance) (:id instance) (ex-message e))))))
-
-(def analyzable-entities
- "Entities for which we can compute analysis findings."
- #{:card :transform :segment})
-
-(def ^:private AnalyzableEntityType
- "Schema for entity types supported by analysis findings."
- (into [:enum] analyzable-entities))
-
-(def dependable-entities
- "Entities which can be depended on by other entities, even if they cannot themselves be analyzed.
-
- For example, `:table`s can be passed to [[mark-transitive-dependents-stale!]] as an upstream entity whose dependents are now
- stale, but we cannot analyze tables themselves."
- (conj analyzable-entities :table))
-
-(defn- mark-supported-dependents-stale!
- "Given a map of {entity-type [entity-ids]}, mark all supported (analyzable) types as stale.
- For non-analyzable entities (e.g., tables), looks through to their dependents so we still
- reach analyzable entities beyond them.
- Returns true if any supported dependents were found."
- [dependents]
- (let [{analyzable true non-analyzable false}
- (group-by (fn [[dep-type _]] (boolean (analyzable-entities dep-type)))
- (filter (fn [[_ dep-ids]] (seq dep-ids)) dependents))
- ;; For non-analyzable entities, find their direct dependents
- pass-through-dependents
- (when (seq non-analyzable)
- (let [key-seq (for [[dep-type dep-ids] non-analyzable
- id dep-ids]
- [dep-type id])
- deps-map (models.dependency/direct-dependents key-seq)]
- (models.dependency/group-nodes (into #{} cat (vals deps-map)))))
- ;; Merge the analyzable dependents with the pass-through dependents
- all-supported (merge-with into
- (into {} analyzable)
- (into {}
- (filter (fn [[dep-type _]] (analyzable-entities dep-type)))
- pass-through-dependents))]
- (deps.analysis-finding/mark-stale! all-supported)
- (boolean (seq all-supported))))
-
-(defn mark-transitive-dependents-stale!
- "Mark all transitive dependents of the given upstream entities stale for re-analysis. Takes `{entity-type [entity-ids]}`
- — entity types may be non-analyzable (e.g. `:table`), since they can still have analyzable dependents, and each id
- collection may be any seqable (vector, set, ...). Returns true if any supported (analyzable) dependents were found.
-
- Uses `transitive-dependents` (not `transitive-mbql-dependents`) because:
- 1. It's faster - doesn't need to fetch entities to check for native-ness
- 2. Native dependents marked stale will just be quickly re-validated as valid"
- [type->entity-ids]
- (-> (models.dependency/transitive-dependents
- (update-vals type->entity-ids (fn [ids] (map (fn [id] {:id id}) ids))))
- mark-supported-dependents-stale!))
-
-(mu/defn mark-entity-stale!
- "Mark a single entity as stale for re-analysis by the background job."
- [entity-type :- AnalyzableEntityType
- entity-id :- pos-int?]
- (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))
-
-(mu/defn mark-entity-and-transitive-dependents-stale!
- "Mark `entity-id` and all of its transitive dependents stale for re-analysis. Use this when an entity changes: it
- queues the whole affected subtree up front so the entity-check job drains a fixed set of stale entities (see
- [[metabase-enterprise.dependencies.task.entity-check/check-entities!]]) without re-propagating staleness during the
- drain — which can never terminate when the dependency graph has a cycle."
- [entity-type :- AnalyzableEntityType
- entity-id :- pos-int?]
- (mark-entity-stale! entity-type entity-id)
- (mark-transitive-dependents-stale! {entity-type [entity-id]}))
-
-(mu/defn analyze-batch! :- nat-int?
- "Add or update analyses for a batch of entities.
-
- Takes in an entity type and batch size, and looks for a batch of entities with missing or out of date
- AnalysisFindings and then upsert new analyses for them.
-
- This only *analyzes* (each [[upsert-analysis!]] clears that entity's stale flag); it does NOT propagate staleness to
- dependents. Propagation happens up front when an entity changes (see [[mark-entity-and-transitive-dependents-stale!]] and the
- backfill job), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeing
- termination even when the dependency graph has cycles."
- [type :- AnalyzableEntityType
- batch-size :- pos-int?]
- ;; The cache spans the select too: reading an entity attaches a `MetadataProvider` to its query
- ;; (see `metabase.lib-be.models.transforms/transform-query`), so selecting outside the cache gives every entity in
- ;; the batch a private one.
- (lib-be/with-metadata-provider-cache
- (let [instances (deps.analysis-finding/instances-for-analysis type batch-size)]
- (doseq [instance instances]
- (try (upsert-analysis! instance)
- (catch Exception e
- (log/errorf "Analyzing entity %s %s failed: %s"
- (t2/model instance) (:id instance) (ex-message e)))))
- (count instances))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
index 9b042052562..d2453f9d283 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
@@ -1,253 +1,326 @@
(ns metabase-enterprise.dependencies.hooks
- "Implements the hooks of `metabase.dependencies.hooks`, marking written content for the background jobs to recompute."
+ "Implements the hooks of `metabase.dependencies.hooks`: a write marks the statuses it makes stale in its own
+ transaction, and triggers the background jobs once it commits."
(:require
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.task.backfill :as task.backfill]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]
+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]
+ [metabase-enterprise.dependencies.task.dependency :as task.dependency]
+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]
[metabase.app-db.core :as mdb]
[metabase.batch-processing.core :as grouper]
- [metabase.models.interface :as mi]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn- trigger-jobs-now!
- "Trigger the backfill and entity-check jobs, logging rather than throwing a failure."
+(def ^:private job-triggers
+ "The functions that trigger the jobs: the dependency job, the analysis finding job, and the result metadata job."
+ [#'task.dependency/trigger-dependency-job!
+ #'task.analysis-finding/trigger-analysis-finding-job!
+ #'task.result-metadata/trigger-result-metadata-job!])
+
+(mu/defn- trigger-jobs-now!
+ "Trigger every job, logging rather than throwing a failure."
[]
- (try
- (task.backfill/trigger-backfill-job!)
- (task.entity-check/trigger-entity-check-job!)
- (catch Throwable e
- (log/warnf "Failed to trigger the dependency jobs: %s" (ex-message e)))))
-
-(defn- process-writes!
- "Mark the dependents of the Tables changed by `writes` for re-analysis, then trigger the jobs."
- [writes]
- (when-let [table-ids (not-empty (into #{} (keep :table-id) writes))]
- (deps.findings/mark-transitive-dependents-stale! {:table table-ids}))
- (trigger-jobs-now!))
+ (doseq [trigger! job-triggers]
+ (try
+ (trigger!)
+ (catch Throwable e
+ (log/warnf "Failed to trigger %s: %s" (name (symbol trigger!)) (ex-message e))))))
(def ^:private writes-queue
- (delay (grouper/start! process-writes! :capacity 500 :interval 1000)))
+ (delay (grouper/start! (fn [_writes] (trigger-jobs-now!)) :capacity 500 :interval 1000)))
-(defn- submit-after-commit!
- "Submit `write` to the batch processed once the current transaction commits."
- [write]
- (mdb/do-after-commit #(grouper/submit! @writes-queue write)))
+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
+(def ^:private ^:dynamic *batched-marks*
+ "Within [[do-with-batched-marks]], an atom of what the batch marks when it ends: the `:dependents` of
+ `{entity-type #{entity-id}}`, the `:remapped-tables` and the `:remap-targets` fields, the only way the hooks of a
+ batch can mark each of them once."
+ nil)
-(defn- trigger-jobs!
- "Have the backfill and entity-check jobs triggered for this write, once it commits, together with the others in its
- batch, unless a serdes load triggers them once it finishes."
+(mu/defn- trigger-jobs!
+ "Have every job triggered once the current transaction commits, together with the other writes in its batch."
[]
- (when-not mi/*deserializing?*
- (submit-after-commit! :ok)))
-
-(defn- mark-changed!
- "Mark the entity `entity-type` `entity-id` for recalculation and, when analyzable, for re-analysis; the backfill job
- marks its dependents for re-analysis."
- [entity-type entity-id]
- (when entity-id
- (deps.dependency-status/mark-stale! {entity-type #{entity-id}})
- (when (contains? deps.findings/analyzable-entities entity-type)
- (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))
- (trigger-jobs!)))
-
-(defn- changed?
- "Whether the Toucan update of `instance` changes any of `columns`."
- [instance columns]
- (let [changes (t2/changes instance)]
- (some #(contains? changes %) columns)))
-
-(defn- mark-entity-changed!
- "Mark the `model` entity `instance` for recalculation when its update changes any of `columns`."
- [model instance columns]
- (when (changed? instance columns)
- (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))
-
-(defmulti ^:private before-insert*
- "Mark what inserting the `model` row `instance` affects."
- {:arglists '([model instance])}
- (fn [model _instance] model))
-
-(defmulti ^:private before-update*
- "Mark what updating the `model` row `instance` affects."
- {:arglists '([model instance])}
- (fn [model _instance] model))
-
-(defmulti ^:private before-delete*
- "Mark what deleting the `model` row `instance` affects."
- {:arglists '([model instance])}
- (fn [model _instance] model))
-
-(defmethod before-insert* :default [_model _instance] nil)
-(defmethod before-update* :default [_model _instance] nil)
-(defmethod before-delete* :default [_model _instance] nil)
-
-(doseq [model [:model/Action :model/Card :model/Dashboard :model/Document :model/Measure :model/NativeQuerySnippet
- :model/Sandbox :model/Segment :model/Transform]]
- (defmethod before-insert* model
- [_model _instance]
- (trigger-jobs!))
- (defmethod before-delete* model
- [model instance]
- (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))
-
-(doseq [[model columns] {:model/Action #{:model_id :type}
- :model/Card #{:type :dataset_query :dimension_mappings :parameters}
- :model/Dashboard #{:parameters}
- :model/Document #{:content_type :document}
- :model/Measure #{:definition :table_id}
- :model/NativeQuerySnippet #{:content :template_tags}
- :model/Sandbox #{:card_id}
- :model/Segment #{:definition :table_id}
- :model/Transform #{:source :target}}]
- (defmethod before-update* model
- [model instance]
- (mark-entity-changed! model instance columns)))
-
-(defmethod before-insert* :model/QueryAction
- [_model query-action]
- (mark-changed! :action (:action_id query-action)))
-
-(defmethod before-update* :model/QueryAction
- [_model query-action]
- (when (changed? query-action #{:dataset_query})
- (mark-changed! :action (:action_id query-action))))
-
-(defmethod before-insert* :model/DashboardCard
- [_model dashcard]
- (mark-changed! :dashboard (:dashboard_id dashcard)))
-
-(defmethod before-update* :model/DashboardCard
- [_model dashcard]
- (when (changed? dashcard #{:card_id :dashboard_id :visualization_settings})
- (doseq [dashboard-id (distinct [(:dashboard_id (t2/original dashcard)) (:dashboard_id dashcard)])]
- (mark-changed! :dashboard dashboard-id))))
-
-(defmethod before-delete* :model/DashboardCard
- [_model dashcard]
- (mark-changed! :dashboard (:dashboard_id dashcard)))
-
-(defmethod before-insert* :model/DashboardCardSeries
- [_model series]
- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))
-
-(defmethod before-update* :model/DashboardCardSeries
- [_model series]
- (when (changed? series #{:card_id :dashboardcard_id})
- (doseq [dashcard-id (distinct [(:dashboardcard_id (t2/original series)) (:dashboardcard_id series)])]
- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id dashcard-id)))))
-
-(defmethod before-delete* :model/DashboardCardSeries
- [_model series]
- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))
-
-(defmethod before-delete* :model/DashboardTab
- [_model tab]
- (mark-changed! :dashboard (:dashboard_id tab)))
-
-(defn- mark-table-changed!
- "Mark the dependents of the Table with `table-id` for re-analysis once the current transaction commits."
- [table-id]
- (when table-id
- (submit-after-commit! {:table-id table-id})))
-
-(defmethod before-insert* :model/TableUserSettings
- [_model settings]
- (mark-table-changed! (:table_id settings)))
-
-(defmethod before-update* :model/TableUserSettings
- [_model settings]
- (mark-table-changed! (:table_id settings)))
-
-(defmethod before-delete* :model/TableUserSettings
- [_model settings]
- (mark-table-changed! (:table_id settings)))
-
-(defmethod before-insert* :model/FieldUserSettings
- [_model settings]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
-
-(defmethod before-update* :model/FieldUserSettings
- [_model settings]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
-
-(defmethod before-delete* :model/FieldUserSettings
- [_model settings]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
-
-(defmethod before-insert* :model/Dimension
- [_model dimension]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
-
-(defmethod before-update* :model/Dimension
- [_model dimension]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
-
-(defmethod before-delete* :model/Dimension
- [_model dimension]
- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
-
-(defmethod before-delete* :model/Database
- [_model database]
- (doseq [transform-id (dependencies.db/transform-ids-of-source-database (:id database))]
- (mark-changed! :transform transform-id)))
+ (when-not *batched-marks*
+ (mdb/do-after-commit #(grouper/submit! @writes-queue {:dependency-jobs true}))))
+
+(def ^:private Row
+ "A row a hook is given, a Toucan instance."
+ [:fn {:error/message "a row"} map?])
+
+(def ^:private Columns
+ "The columns a write of a model has to change, or `:any`."
+ [:maybe [:or [:= :any] [:set :keyword]]])
+
+(def ^:private Node
+ "An entity as `[entity-type entity-id]`, the ID nil when unknown."
+ [:maybe [:tuple ::deps.dependency-types/dependency-types [:maybe ms/PositiveInt]]])
+
+(mu/defn- changed? :- :boolean
+ "Whether the Toucan update of `row` changes any of `columns`, or any column at all when `columns` is `:any`."
+ [row :- Row
+ columns :- Columns]
+ (let [changes (t2/changes row)]
+ (if (= :any columns)
+ (boolean (seq changes))
+ (boolean (some #(contains? changes %) columns)))))
+
+(def ^:private dependency-columns
+ "For each model, the columns whose update has the dependency job compute the entity's dependencies again."
+ {:model/Action #{:model_id :type}
+ :model/Card #{:type :dataset_query :dimension_mappings :parameters}
+ :model/Dashboard #{:parameters}
+ :model/Document #{:content_type :document}
+ :model/Measure #{:definition :table_id}
+ :model/NativeQuerySnippet #{:content :template_tags}
+ :model/Sandbox #{:card_id}
+ :model/Segment #{:definition :table_id}
+ :model/Transform #{:source :target}})
+
+(def ^:private analysis-columns
+ "For each model, the columns whose update has the analysis job analyze the entity itself again."
+ {:model/Card #{:type :dataset_query}
+ :model/Segment #{:definition :table_id}
+ :model/Transform #{:source :target}})
+
+(def ^:private model-output-columns
+ "For each model, the columns whose update changes what the entities directly depending on it read, so their statuses
+ are marked."
+ {:model/Card #{:type :result_metadata}
+ :model/Measure #{:definition :table_id}
+ :model/NativeQuerySnippet #{:content :template_tags}
+ :model/Segment #{:definition :table_id}
+ :model/Table #{:name :schema :display_name :active :visibility_type}})
+
+(mu/defn- output-columns :- Columns
+ "The [[model-output-columns]] of `row` of `model`, with the name of a metric, which the cards aggregating it show."
+ [model :- :keyword
+ row :- Row]
+ (cond-> (get model-output-columns model)
+ (and (= :model/Card model) (= :metric (keyword (:type row)))) (conj :name)))
+
+(def ^:private field-columns
+ "The Field columns whose update changes what the entities using the Field's Table read."
+ #{:name :display_name :description :active :visibility_type :settings :base_type :effective_type :database_type
+ :semantic_type :coercion_strategy :fk_target_field_id :position})
+
+(mu/defn- part-owner :- Node
+ "The entity, as `[entity-type entity-id]`, that `row` of `model` is a part of, or nil."
+ [model :- :keyword
+ row :- [:maybe Row]]
+ (case model
+ (:model/DashboardCard :model/DashboardTab) [:dashboard (:dashboard_id row)]
+ :model/DashboardCardSeries [:dashboard (some-> (:dashboardcard_id row)
+ dependencies.db/dashcard-dashboard-id)]
+ :model/QueryAction [:action (:action_id row)]
+ (:model/Field :model/TableUserSettings) [:table (:table_id row)]
+ :model/FieldUserSettings [:table (some-> (:field_id row) dependencies.db/field-table-id)]
+ nil))
+
+(def ^:private part-columns
+ "For each model whose rows are parts of an entity, the columns whose update changes the entity."
+ {:model/DashboardCard #{:card_id :dashboard_id :visualization_settings}
+ :model/DashboardCardSeries #{:card_id :dashboardcard_id}
+ :model/DashboardTab #{}
+ :model/QueryAction #{:dataset_query}
+ :model/Field field-columns
+ :model/TableUserSettings #{:display_name :visibility_type :data_layer :field_order}
+ :model/FieldUserSettings (conj field-columns :custom_position :nfc_path :json_unfolding)})
+
+(mu/defn- mark-dependents!
+ "Mark the result metadata and analysis statuses of the entities directly depending on the entity `entity-type`
+ `entity-id` stale."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity-id :- ms/PositiveInt]
+ (if-let [batched *batched-marks*]
+ (swap! batched update-in [:dependents entity-type] (fnil conj #{}) entity-id)
+ (do
+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! entity-type #{entity-id})
+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! entity-type #{entity-id}))))
+
+(mu/defn- mark-remapped-tables!
+ "Mark the result metadata and analysis statuses of every entity transitively depending on the Tables with `table-ids`
+ stale, whose remapped columns their stored metadata carries down."
+ [table-ids :- [:set ms/PositiveInt]]
+ (when (seq table-ids)
+ (if-let [batched *batched-marks*]
+ (swap! batched update :remapped-tables (fnil into #{}) table-ids)
+ (let [dependents (models.dependency/transitive-dependents {:table (mapv #(hash-map :id %) table-ids)})]
+ (dependencies.db/mark-result-metadata-statuses-stale! dependents)
+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents)))))
+
+(mu/defn- mark-remap-targets!
+ "Mark what reads the remaps whose human-readable field is one of the Fields with `field-ids`, a change of which the
+ remapped columns show."
+ [field-ids :- [:set ms/PositiveInt]]
+ (when (seq field-ids)
+ (if-let [batched *batched-marks*]
+ (swap! batched update :remap-targets (fnil into #{}) field-ids)
+ (mark-remapped-tables! (dependencies.db/remapped-table-ids field-ids)))))
+
+(mu/defn- mark-dimension!
+ "Mark what reads the remap of the Dimension `row`, whose Field's Table's entities show it."
+ [row :- [:maybe Row]]
+ (when-let [table-id (some-> (:field_id row) dependencies.db/field-table-id)]
+ (mark-remapped-tables! #{table-id})))
+
+(mu/defn- mark-field-values! :- :boolean
+ "Mark what reads the internal remap of the Field of the FieldValues, which shows only while they have human-readable
+ values, when a write of them as `before` and `after` adds or removes those."
+ [before :- [:maybe Row]
+ after :- [:maybe Row]]
+ (let [row (or after before)
+ readable (comp boolean seq :human_readable_values)]
+ (boolean
+ (when (and (= :full (keyword (:type row)))
+ (not= (readable before) (readable after))
+ (dependencies.db/internal-remap? (:field_id row)))
+ (mark-dimension! row)
+ true))))
+
+(mu/defn- mark-output-changed!
+ "Mark what reads the output of the entity `entity-type` `entity` stale: the direct dependents, or every transitive
+ dependent of a measure or a snippet, which their dependents expand and nothing between them stores."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity :- Row]
+ (case entity-type
+ :measure (let [dependents (models.dependency/transitive-dependents {:measure [entity]})]
+ (dependencies.db/mark-result-metadata-statuses-stale! dependents)
+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents))
+ :snippet (let [dependents (models.dependency/transitive-dependents {:snippet [entity]})]
+ (dependencies.db/mark-dependency-statuses-stale! dependents)
+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents))
+ (mark-dependents! entity-type (:id entity))))
+
+(mu/defn- mark-part-owner!
+ "Mark what a write of a part of the `[owner-type owner-id]` entity changes: the dependencies of a dashboard or an
+ action, or what the entities depending on a table read."
+ [[owner-type owner-id] :- Node]
+ (when owner-id
+ (if (= :table owner-type)
+ (mark-dependents! :table owner-id)
+ (dependencies.db/mark-dependency-statuses-stale! {owner-type #{owner-id}}))))
+
+(mu/defn- output-changed? :- :boolean
+ "Whether the update of `row` of `model` changes what the entities directly depending on it read, ignoring a card's
+ result metadata changes they don't see, such as a run's new fingerprints."
+ [model :- :keyword
+ row :- Row]
+ (let [columns (output-columns model row)]
+ (and (changed? row columns)
+ (or (not= :model/Card model)
+ (changed? row (disj columns :result_metadata))
+ (not (deps.result-metadata/same-output? (:result_metadata (t2/original row)) (:result_metadata row)))))))
+
+(mu/defn- mark-update! :- :boolean
+ "Mark the statuses an update of `row` of `model` makes stale: the entity's own, by the columns each job
+ reads, and what reads its output when that changes, returning whether a column some job reads changed."
+ [model :- :keyword
+ row :- Row]
+ (let [entity-type (deps.dependency-types/model->dependency-type model)
+ own {entity-type #{(:id row)}}
+ dependency? (changed? row (dependency-columns model))
+ analysis? (changed? row (analysis-columns model))
+ output? (output-changed? model row)
+ part? (changed? row (part-columns model))]
+ (when dependency?
+ (dependencies.db/mark-dependency-statuses-stale! own))
+ (when analysis?
+ (dependencies.db/mark-analysis-finding-statuses-stale! own))
+ (when output?
+ (mark-output-changed! entity-type row))
+ (when part?
+ (run! mark-part-owner! (distinct [(part-owner model (t2/original row)) (part-owner model row)])))
+ (when part?
+ (case model
+ :model/Field (mark-remap-targets! #{(:id row)})
+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})
+ nil))
+ (or dependency? analysis? output? part?
+ (case model
+ :model/Dimension (do (run! mark-dimension! (distinct [(t2/original row) row])) true)
+ :model/FieldValues (mark-field-values! (t2/original row) row)
+ false))))
+
+(defenterprise do-with-batched-marks
+ "Run `thunk`, marking what its writes change once per entity after it ends, even when it throws, instead of once per
+ row in each write's transaction, and triggering the jobs once."
+ :feature :dependencies
+ [thunk]
+ (if *batched-marks*
+ (thunk)
+ (let [batched (atom {})]
+ (try
+ (binding [*batched-marks* batched]
+ (thunk))
+ (finally
+ (let [{:keys [dependents remapped-tables remap-targets]} @batched]
+ (doseq [[entity-type entity-ids] dependents]
+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! entity-type entity-ids)
+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! entity-type entity-ids))
+ (mark-remapped-tables! (into (set remapped-tables)
+ (when (seq remap-targets)
+ (dependencies.db/remapped-table-ids remap-targets)))))
+ (trigger-jobs!))))))
(defenterprise before-insert
- "Mark what inserting the `model` row `instance` affects for recalculation."
+ "Mark what inserting `row` of `model` changes, and trigger the jobs once it commits; a new entity has no
+ status, which has the jobs process it."
:feature :dependencies
- [model instance]
- (before-insert* model instance))
+ [model row]
+ (when (contains? part-columns model)
+ (mark-part-owner! (part-owner model row)))
+ (case model
+ :model/Dimension (mark-dimension! row)
+ :model/FieldValues (mark-field-values! nil row)
+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})
+ nil)
+ (trigger-jobs!))
(defenterprise before-update
- "Mark what updating the `model` row `instance` affects for recalculation."
+ "Mark what updating `row` of `model` changes, and trigger the jobs once it commits when it marked any."
:feature :dependencies
- [model instance]
- (before-update* model instance))
+ [model row]
+ (when (mark-update! model row)
+ (trigger-jobs!)))
(defenterprise before-delete
- "Mark what deleting the `model` row `instance` affects for recalculation."
- :feature :dependencies
- [model instance]
- (before-delete* model instance))
-
-(defenterprise after-database-sync
- "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed, and the
- entities reading from it that name tables that matched nothing, which sync may have found now."
+ "Mark what deleting `row` of `model` changes: the entity it is a part of, what reads a deleted entity, or
+ the transforms reading a deleted Database; and trigger the jobs once it commits."
:feature :dependencies
- [database-id]
- (doseq [[entity-type entity-ids] (dependencies.db/unresolved-table-entity-ids database-id)
- entity-id entity-ids]
- (mark-changed! entity-type entity-id))
- (when (deps.findings/mark-transitive-dependents-stale!
- {:table (dependencies.db/table-ids-with-outdated-findings database-id)})
- (trigger-jobs!)))
+ [model row]
+ (when (contains? part-columns model)
+ (mark-part-owner! (part-owner model row)))
+ (case model
+ :model/Dimension (mark-dimension! row)
+ :model/FieldValues (mark-field-values! row nil)
+ :model/Field (mark-remap-targets! #{(:id row)})
+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})
+ nil)
+ (when-let [entity-type (deps.dependency-types/model->dependency-type model)]
+ (mark-output-changed! entity-type row))
+ (when (= model :model/Database)
+ (when-let [transform-ids (not-empty (dependencies.db/source-database-transform-ids (:id row)))]
+ (dependencies.db/mark-dependency-statuses-stale! {:transform transform-ids})
+ (dependencies.db/mark-analysis-finding-statuses-stale! {:transform transform-ids})))
+ (trigger-jobs!))
(defenterprise after-transform-run
- "Make the output Table with `table-id` depend on the transform with `transform-id`, and mark the transform's
- dependents for re-analysis."
+ "Make the output Table with `table-id` depend on the transform with `transform-id`."
:feature :dependencies
[transform-id table-id]
(when table-id
(models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))
- (when (deps.findings/mark-transitive-dependents-stale! {:transform #{transform-id}})
- (trigger-jobs!)))
-
-(defenterprise after-serdes-load
- "Trigger the backfill and entity-check jobs for what a serdes load marked."
- :feature :dependencies
- []
- (trigger-jobs-now!))
+ (trigger-jobs!))
(defenterprise after-premium-token-update
- "Restart the backfill and entity-check jobs, which stop while the dependencies feature is disabled."
+ "Restart every job, since they stop while the dependencies feature is disabled."
:feature :dependencies
[]
(trigger-jobs-now!))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
index 29087660a17..f6637ad3a92 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
@@ -3,5 +3,6 @@
[metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.schema]
[metabase-enterprise.dependencies.settings]
- [metabase-enterprise.dependencies.task.backfill]
- [metabase-enterprise.dependencies.task.entity-check]))
+ [metabase-enterprise.dependencies.task.analysis-finding]
+ [metabase-enterprise.dependencies.task.dependency]
+ [metabase-enterprise.dependencies.task.result-metadata]))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
index 0a3dfbbd278..30df6d1906e 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
@@ -2,8 +2,11 @@
(:require
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]
+ [metabase.lib.schema.validate :as lib.schema.validate]
[metabase.models.interface :as mi]
[metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -12,7 +15,7 @@
(derive :model/AnalysisFinding :metabase/model)
(t2/deftransforms :model/AnalysisFinding
- {:analyzed_entity_type mi/transform-keyword})
+ {:entity_type mi/transform-keyword})
(t2/define-before-insert :model/AnalysisFinding
[finding]
@@ -42,61 +45,30 @@
- 8: Flag MBQL queries whose source-table is missing/inactive (e.g. dropped during sync)"
8)
-(defn- error->finding-error-row
+(mu/defn- error->finding-error-row :- ::deps.analysis-finding-error/error-input
"Convert an error from lib/find-bad-refs-with-source to a row for analysis_finding_error table.
We use `:message` for the error detail if there was an `validation-exception-error`."
- [error]
+ [error :- ::lib.schema.validate/error-with-source]
{:error-type (:type error)
;; error-detail cannot be longer than 254
:error-detail (some-> (or (:name error) (:message error)) (u/truncate 254))
:source-entity-type (:source-entity-type error)
:source-entity-id (:source-entity-id error)})
-(defn upsert-analysis!
+(def ^:private FindingDetails
+ "The errors an analysis found, as `lib/find-bad-refs-with-source` returns them."
+ [:maybe [:set ::lib.schema.validate/error-with-source]])
+
+(mu/defn upsert-analysis!
"Given the details of an AnalysisFinding row, upsert the data into the actual db.
Also writes individual errors to the analysis_finding_error table with source information."
- [type instance-id result finding-details]
+ [entity-type :- :keyword
+ entity-id :- ms/PositiveInt
+ result :- :boolean
+ finding-details :- FindingDetails]
(t2/with-transaction [_conn]
- (let [update {:analysis_version *current-analysis-finding-version*
- :result result
- :stale false}
- existing-id (dependencies.db/finding-id type instance-id)]
- (if existing-id
- (dependencies.db/update-finding! existing-id update)
- (dependencies.db/insert-finding! (assoc update
- :analyzed_entity_type type
- :analyzed_entity_id instance-id)))
- (deps.analysis-finding-error/replace-errors-for-entity!
- type
- instance-id
- (map error->finding-error-row finding-details)))))
-
-(def ^:private mark-stale-batch-size
- "Maximum number of entity IDs to process in a single query to avoid parameter limits."
- 1000)
-
-(defn mark-stale!
- "Mark the existing findings of the entities in `type->ids`, `{entity-type ids}`, stale for re-analysis. Entities
- without findings are ignored - they'll be picked up by the normal job flow."
- [type->ids]
- (doseq [[entity-type ids] type->ids
- batch (partition-all mark-stale-batch-size ids)]
- (dependencies.db/mark-findings-stale! entity-type batch)))
-
-(defn has-stale-entities?
- "Check if there are any stale analysis records."
- []
- (dependencies.db/stale-finding-exists?))
-
-(defn stale-entity-count
- "Number of analysis findings currently marked stale, across all entity types. Used by the entity-check drain loop to
- detect whether it is still making progress."
- []
- (dependencies.db/stale-finding-count))
-
-(defn instances-for-analysis
- "Find a batch of instances of type `entity-type` and maximum size `batch-size` with missing, outdated,
- or stale AnalysisFindings.
- Prioritizes stale entities (stale=true) over outdated version entities."
- [entity-type batch-size]
- (dependencies.db/instances-for-analysis entity-type batch-size *current-analysis-finding-version*))
+ (dependencies.db/upsert-finding! entity-type entity-id result)
+ (deps.analysis-finding-error/replace-errors-for-entity!
+ entity-type
+ entity-id
+ (map error->finding-error-row finding-details))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.clj
new file mode 100644
index 00000000000..c4b0b88b2ca
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.clj
@@ -0,0 +1,13 @@
+(ns metabase-enterprise.dependencies.models.analysis-finding-status
+ "The AnalysisFindingStatus model: one status per analyzable entity, marked stale by the writes the analysis reads."
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/AnalysisFindingStatus [_model] :analysis_finding_status)
+
+(derive :model/AnalysisFindingStatus :metabase/model)
+
+(t2/deftransforms :model/AnalysisFindingStatus
+ {:entity_type mi/transform-keyword})
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 5812611cb8f..da302e456f7 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.schema :as dependencies.schema]
[metabase-enterprise.sandbox.schema]
[metabase.actions.schema :as actions.schema]
[metabase.documents.schema :as documents.schema]
@@ -17,6 +18,8 @@
[metabase.transforms.schema :as transforms.schema]
[metabase.util :as u]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[potemkin :as p]
[toucan2.core :as t2]))
@@ -44,7 +47,35 @@
{:from_entity_type mi/transform-keyword
:to_entity_type mi/transform-keyword})
-(defn- deps-children
+(mr/def ::entity-with-id
+ "Any map with an optional `:id`, whatever else it holds."
+ [:map {::mr/deliberately-open true, :description "a Toucan, metadata, or user-supplied entity"}
+ [:id {:optional true} [:maybe ::deps.dependency-types/entity-id]]])
+
+(def ^:private EntitiesMap
+ "Entities as `{entity-type [entity-or-id]}`, nil for none of a type: each one an entity ID, or a map whose `:id` is
+ one (skipped when nil), such as a Toucan entity, a `MetadataProvider` entity, or user input."
+ [:map-of
+ ::deps.dependency-types/dependency-types
+ [:maybe
+ [:or
+ [:sequential [:or ::deps.dependency-types/entity-id ::entity-with-id]]
+ [:set [:or ::deps.dependency-types/entity-id ::entity-with-id]]]]])
+
+(def ^:private UncheckedEntityId
+ "A dependency entity ID as a caller gives it, which [[ensure-entity-id]] checks."
+ [:schema {::mr/deliberately-open true, :description "an unchecked dependency entity id"} :any])
+
+(def ^:private DependenciesByType
+ "The dependencies of an entity as `{entity-type #{entity-id}}`, with the IDs not yet checked."
+ [:map-of ::dependencies.schema/entity-type [:set UncheckedEntityId]])
+
+(def ^:private KeyFn
+ "[[key-dependents]] or [[key-dependencies]], called with nodes and the two edge restrictions."
+ [:-> [:sequential ::dependencies.schema/node] ::dependencies.schema/edge-restriction ::dependencies.schema/edge-restriction
+ ::dependencies.schema/child-map])
+
+(mu/defn- deps-children :- ::dependencies.schema/child-map
"Get dependency children with optional visibility filtering.
Returns a map from [src-type src-id] tuples to sets of [dst-type dst-id] tuples representing dependencies.
@@ -53,20 +84,28 @@
`edge-restriction-expr`: nil, `{:visible {...}}`, `{:broken? true}`, `{:types #{...}}`, or
`{:entity-type ... :ids #{...}}`) restricting the destination (child) and source (parent) side of each edge,
respectively."
- ([{:keys [src-type src-id dst-type dst-id key-seq destination-filter source-filter]}]
- (transduce (map (fn [[entity-type entity-keys]]
- (let [deps (dependencies.db/dependency-edges
- {:src-type src-type :src-id src-id :dst-type dst-type :dst-id dst-id
- :entity-type entity-type :entity-ids entity-keys
- :destination-restriction destination-filter :source-restriction source-filter})]
- (u/group-by (juxt src-type src-id)
- (juxt dst-type dst-id)
- conj #{}
- deps))))
- merge {}
- (u/group-by first second key-seq))))
-
-(defn- key-dependents
+ [{:keys [src-type src-id dst-type dst-id key-seq destination-filter source-filter]}
+ :- [:map {:closed true}
+ [:src-type [:enum :from_entity_type :to_entity_type]]
+ [:src-id [:enum :from_entity_id :to_entity_id]]
+ [:dst-type [:enum :from_entity_type :to_entity_type]]
+ [:dst-id [:enum :from_entity_id :to_entity_id]]
+ [:key-seq [:sequential ::dependencies.schema/node]]
+ [:destination-filter ::dependencies.schema/edge-restriction]
+ [:source-filter ::dependencies.schema/edge-restriction]]]
+ (transduce (map (fn [[entity-type entity-keys]]
+ (let [deps (dependencies.db/dependency-edges
+ {:src-type src-type :src-id src-id :dst-type dst-type :dst-id dst-id
+ :entity-type entity-type :entity-ids entity-keys
+ :destination-restriction destination-filter :source-restriction source-filter})]
+ (u/group-by (juxt src-type src-id)
+ (juxt dst-type dst-id)
+ conj #{}
+ deps))))
+ merge {}
+ (u/group-by first second key-seq)))
+
+(mu/defn- key-dependents :- ::dependencies.schema/child-map
"Get the dependent entity keys for the entity keys in `key-seq`.
Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys
@@ -74,9 +113,11 @@
`destination-filter` and `source-filter` are filter specs restricting the dependent (child) and queried (parent)
side of each edge, respectively — see [[deps-children]]."
- ([key-seq]
+ ([key-seq :- [:sequential ::dependencies.schema/node]]
(key-dependents key-seq nil nil))
- ([key-seq destination-filter source-filter]
+ ([key-seq :- [:sequential ::dependencies.schema/node]
+ destination-filter :- ::dependencies.schema/edge-restriction
+ source-filter :- ::dependencies.schema/edge-restriction]
(deps-children
{:src-type :to_entity_type
:src-id :to_entity_id
@@ -86,12 +127,12 @@
:destination-filter destination-filter
:source-filter source-filter})))
-(defn direct-dependents
+(mu/defn direct-dependents :- ::dependencies.schema/child-map
"Returns the direct dependents for the given entity key sequence."
- [key-seq]
+ [key-seq :- [:sequential ::dependencies.schema/node]]
(key-dependents key-seq))
-(defn- key-dependencies
+(mu/defn- key-dependencies :- ::dependencies.schema/child-map
"Get the dependency entity keys for the entity keys in `key-seq`.
Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys
@@ -99,9 +140,11 @@
`destination-filter` and `source-filter` are filter specs restricting the dependency (child) and queried (parent)
side of each edge, respectively — see [[deps-children]]."
- ([key-seq]
+ ([key-seq :- [:sequential ::dependencies.schema/node]]
(key-dependencies key-seq nil nil))
- ([key-seq destination-filter source-filter]
+ ([key-seq :- [:sequential ::dependencies.schema/node]
+ destination-filter :- ::dependencies.schema/edge-restriction
+ source-filter :- ::dependencies.schema/edge-restriction]
(deps-children
{:src-type :from_entity_type
:src-id :from_entity_id
@@ -114,56 +157,60 @@
(p/deftype+ DependencyGraph [children-fn]
graph/Graph
(children-of [_this key-seq]
- (children-fn key-seq)))
+ (children-fn (vec key-seq))))
-(defn graph-dependents
+(mu/defn graph-dependents :- ::graph/graph
"Return a dependency graph for finding dependents (downstream entities)."
[]
(->DependencyGraph key-dependents))
-(defn graph-dependencies
+(mu/defn graph-dependencies :- ::graph/graph
"Return a dependency graph for finding dependencies (upstream entities)."
[]
(->DependencyGraph key-dependencies))
-(defn- filtered-graph
+(mu/defn- filtered-graph :- ::graph/graph
"Create a dependency graph with visibility filtering.
Arguments:
- `key-fn`: Either key-dependencies or key-dependents, determining graph direction
- `destination-filter`: Filter spec restricting the child side of a dependency — see [[deps-children]]
- `source-filter`: Filter spec restricting the parent side of a dependency"
- [key-fn destination-filter source-filter]
+ [key-fn :- KeyFn
+ destination-filter :- ::dependencies.schema/edge-restriction
+ source-filter :- ::dependencies.schema/edge-restriction]
(->DependencyGraph
(fn [key-seq]
(key-fn key-seq destination-filter source-filter))))
-(defn filtered-graph-dependencies
+(mu/defn filtered-graph-dependencies :- ::graph/graph
"Create a permission-aware dependency graph for finding upstream dependencies.
Arguments:
- `destination-filter`: Filter spec restricting destination entities — see [[deps-children]]
- `source-filter`: Optional filter spec restricting the parent side of a dependency"
- ([destination-filter]
+ ([destination-filter :- ::dependencies.schema/edge-restriction]
(filtered-graph-dependencies destination-filter nil))
- ([destination-filter source-filter]
+ ([destination-filter :- ::dependencies.schema/edge-restriction
+ source-filter :- ::dependencies.schema/edge-restriction]
(filtered-graph key-dependencies destination-filter source-filter)))
-(defn filtered-graph-dependents
+(mu/defn filtered-graph-dependents :- ::graph/graph
"Create a permission-aware dependency graph for finding downstream dependents.
Arguments:
- `destination-filter`: Filter spec restricting destination entities — see [[deps-children]]
- `source-filter`: Optional filter spec restricting the parent side of a dependency"
- ([destination-filter]
+ ([destination-filter :- ::dependencies.schema/edge-restriction]
(filtered-graph-dependents destination-filter nil))
- ([destination-filter source-filter]
+ ([destination-filter :- ::dependencies.schema/edge-restriction
+ source-filter :- ::dependencies.schema/edge-restriction]
(filtered-graph key-dependents destination-filter source-filter)))
-(defn entities->nodes
+(mu/defn entities->nodes :- [:sequential ::dependencies.schema/node]
"Converts a map of entities `{entity-type [{:id 1, ...} ...]}` or entity IDs `{entity-type [1]}` into a list of nodes
`[[entity-type entity-id]]`."
- [entities-map]
+ [entities-map :- EntitiesMap]
(for [[entity-type entities] entities-map
entity entities
:let [id (if (number? entity)
@@ -172,12 +219,12 @@
:when id]
[entity-type id]))
-(defn group-nodes
+(mu/defn group-nodes :- ::dependencies.schema/type->ids
"Groups a list of nodes `[[entity-type entity-id]]` by their type."
- [nodes]
+ [nodes :- [:sequential ::dependencies.schema/node]]
(u/group-by first second conj #{} nodes))
-(defn transitive-dependents
+(mu/defn transitive-dependents :- ::dependencies.schema/type->ids
"Given a map of entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
as `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.
@@ -187,8 +234,10 @@
`MetadataProvider` entities, user input, etc.
**Excludes** the input entities from the list of dependents!"
- ([entities-map] (transitive-dependents nil entities-map))
- ([graph entities-map]
+ ([entities-map :- EntitiesMap]
+ (transitive-dependents nil entities-map))
+ ([graph :- [:maybe ::graph/graph]
+ entities-map :- EntitiesMap]
(let [graph (or graph (graph-dependents))
starters (entities->nodes entities-map)]
(->> (graph/transitive graph starters) ; This returns a flat list.
@@ -227,27 +276,33 @@
:action (= :query (:type entity))
false)))
-(defn- native-lookup-map [children]
+(mu/defn- native-lookup-map :- [:map-of ::dependencies.schema/node :boolean]
+ "Whether each node in `children` is native."
+ [children :- ::dependencies.schema/child-map]
(let [grouped (-> (graph/all-map-nodes children)
group-nodes)]
(into {}
(mapcat (fn [[node-type ids]]
- (mapv (fn [entity]
- [[node-type (:id entity)]
- (is-native-entity? node-type entity)])
- (dependencies.db/instances node-type ids))))
+ (if (= node-type :card)
+ (let [native (dependencies.db/native-card-ids ids)]
+ (map (fn [id] [[:card id] (contains? native id)]) ids))
+ (mapv (fn [entity]
+ [[node-type (:id entity)]
+ (is-native-entity? node-type entity)])
+ (dependencies.db/entities node-type ids)))))
grouped)))
-(defn transitive-mbql-dependents
+(mu/defn transitive-mbql-dependents :- ::dependencies.schema/type->ids
"Equivalent to `transitive-dependents`, except it excludes any native cards/transforms/segments and their children.
Also, the order is more flexible (though consistent between runs).
Note that this does not check the passed in entities for native-ness -- the filter is only applied to their
transitive children."
- ([entities-map]
+ ([entities-map :- EntitiesMap]
(transitive-mbql-dependents nil entities-map))
- ([graph entities-map]
+ ([graph :- [:maybe ::graph/graph]
+ entities-map :- EntitiesMap]
(let [start-nodes (set (entities->nodes entities-map))
children (graph/transitive-children-of (or graph (graph-dependents)) (seq start-nodes))
native-lookup (native-lookup-map children)]
@@ -259,17 +314,35 @@
:else node))
children)))))
-(defn- ensure-entity-id
- [id]
+(mu/defn- ensure-entity-id :- ::deps.dependency-types/entity-id
+ "`id`, throwing a 400 when it is not a positive integer."
+ [id :- UncheckedEntityId]
(when-not (pos-int? id)
(throw (ex-info "Dependency entity id must be a positive integer"
{:status-code 400, :id id})))
id)
-(defn replace-dependencies!
- "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with
- the ones specified in `dependencies-by-type`. "
- [entity-type entity-id dependencies-by-type]
+(mu/defn new-dependencies :- [:sequential [:tuple ::dependencies.schema/entity-type ::deps.dependency-types/entity-id]]
+ "The dependencies in `dependencies-by-type` that the entity of type `entity-type` with id `entity-id` doesn't have
+ yet, as `[[entity-type entity-id]]`."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ dependencies-by-type :- DependenciesByType]
+ (let [current (into #{}
+ (map (juxt :to_entity_type :to_entity_id))
+ (dependencies.db/dependencies-from entity-type entity-id))]
+ (vec (for [[to-entity-type ids] dependencies-by-type
+ to-entity-id ids
+ :let [node [to-entity-type (ensure-entity-id to-entity-id)]]
+ :when (not (current node))]
+ node))))
+
+(mu/defn replace-dependencies! :- [:maybe :int]
+ "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with the ones specified in
+ `dependencies-by-type`, returning the number inserted, or nil when none are."
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ dependencies-by-type :- DependenciesByType]
(let [current-dependencies (dependencies.db/dependencies-from entity-type entity-id)
to-remove (keep (fn [{:keys [id to_entity_type to_entity_id]}]
(when-not (get-in dependencies-by-type [to_entity_type to_entity_id])
@@ -289,7 +362,7 @@
(when (seq to-add)
(dependencies.db/insert-dependencies! to-add)))))
-(defn swap-dependency!
+(mu/defn swap-dependency! :- :int
"Efficiently swap a dependency from old-source to new-source during replacement operations.
This is more efficient than full dependency analysis since we know exactly what changed.
@@ -301,7 +374,10 @@
- entity-id: The ID of the entity
- old-source: The source being replaced, as [source-type source-id] (e.g., [:card 783])
- new-source: The new source, as [source-type source-id] (e.g., [:table 164])"
- [entity-type entity-id [old-source-type old-source-id] [new-source-type new-source-id]]
+ [entity-type :- ::dependencies.schema/entity-type
+ entity-id :- ms/PositiveInt
+ [old-source-type old-source-id] :- [:tuple ::dependencies.schema/entity-type ms/PositiveInt]
+ [new-source-type new-source-id] :- [:tuple ::dependencies.schema/entity-type ms/PositiveInt]]
(let [already-present? (dependencies.db/dependency-exists? entity-type entity-id new-source-type new-source-id)]
(if already-present?
(dependencies.db/delete-dependency! entity-type entity-id old-source-type old-source-id)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
index 362646d79eb..47c19f712d1 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
@@ -1,10 +1,12 @@
(ns metabase-enterprise.dependencies.models.dependency-status
(:require
- [java-time.api :as t]
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.schema :as dependencies.schema]
[metabase.models.interface :as mi]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -15,88 +17,49 @@
(t2/deftransforms :model/DependencyStatus
{:entity_type mi/transform-keyword})
-(defn mark-stale!
- "Mark the entities in `type->ids`, `{entity-type ids}`, stale for dependency recalculation, creating their statuses
- if they don't exist and resetting their retry state so previously-failed entities get a fresh chance."
- [type->ids]
- (doseq [[entity-type ids] type->ids
- id ids]
- (dependencies.db/mark-dependency-status-stale! entity-type id)))
-
-(defn upsert-status!
- "Upsert a dependency_status entry, setting stale=false, version to current, whether the entity names tables that
- match nothing, and clearing any failure state."
- [entity-type entity-id unresolved-tables?]
- (dependencies.db/upsert-dependency-status!
- entity-type entity-id models.dependency/current-dependency-analysis-version unresolved-tables?))
-
(defmulti hydrate-for-deps
- "Hydrate a batch of instances with data needed for dependency calculation.
+ "Hydrate a batch of entities with data needed for dependency calculation.
Dispatches on entity-type keyword. Default is identity (no hydration needed)."
- {:arglists '([entity-type instances])}
- (fn [entity-type _instances] entity-type))
+ {:arglists '([entity-type entities])}
+ (fn [entity-type _entities] entity-type))
-(defmethod hydrate-for-deps :default [_ instances] instances)
+(defmethod hydrate-for-deps :default [_ entities] entities)
-(defmethod hydrate-for-deps :dashboard [_ instances]
- (t2/hydrate instances [:dashcards :series]))
+(defmethod hydrate-for-deps :dashboard [_ entities]
+ (t2/hydrate entities [:dashcards :series]))
-(defmethod hydrate-for-deps :action [_ instances]
- (let [queries (when (seq instances)
- (dependencies.db/query-action-queries (mapv :id instances)))]
+(defmethod hydrate-for-deps :action [_ entities]
+ (let [queries (when (seq entities)
+ (dependencies.db/query-action-queries (mapv :id entities)))]
(mapv (fn [{:keys [id] :as action}]
(cond-> action
(contains? queries id) (assoc :dataset_query (queries id))))
- instances)))
+ entities)))
-(defn instances-for-dependency-calculation
- "Find a batch of instances of type `entity-type` and maximum size `batch-size` that need
- dependency calculation: no status row yet, stale=true, OR version < current.
- Excludes terminal entities and entities whose retry delay hasn't elapsed.
- Returns full entity objects. Prioritizes stale over outdated.
- Uses Java time (not DB time) so tests with [[mt/with-clock]] work correctly."
- [entity-type batch-size]
- (dependencies.db/instances-for-dependency-calculation entity-type
- batch-size
- models.dependency/current-dependency-analysis-version
- (t/offset-date-time)))
+(mu/defn take-pending-statuses! :- [:sequential ::dependencies.schema/pending-status]
+ "Up to `batch-size` entities of `entity-type` whose dependencies a write changed, are not computed yet, or were
+ computed by an older analysis, those last, with the `stale` count the job clears afterwards."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (dependencies.db/take-pending-dependency-statuses!
+ entity-type batch-size models.dependency/current-dependency-analysis-version))
-(defn has-pending-retries?
- "Returns true if there are any entities waiting to be retried (not terminal, with a set retry time)."
- []
- (dependencies.db/pending-retry-exists?))
+(mu/defn clear-status! :- :boolean
+ "Record that the dependency job computed the dependencies of the entity `entity-type` `entity-id`, clearing its
+ `stale` count unless a write raised it."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity-id :- ms/PositiveInt
+ stale :- ms/IntGreaterThanOrEqualToZero]
+ (dependencies.db/clear-dependency-status!
+ entity-type entity-id stale models.dependency/current-dependency-analysis-version))
-(defn has-stale-or-outdated?
- "Returns true if there are any entities needing dependency calculation: no status row yet,
- stale=true, OR version < current (not terminal, retry delay elapsed), or deleted entities whose status the backfill
- has yet to clean up. Defined in terms of [[instances-for-dependency-calculation]] so it stays consistent with what
- the backfill processes."
+(mu/defn has-pending-entities? :- :boolean
+ "Returns true if there are any entities whose dependencies are stale, not computed yet or computed by an older
+ analysis, or deleted entities whose status the dependency job has yet to clean up."
[]
(boolean
(some (fn [entity-type]
- (or (seq (instances-for-dependency-calculation entity-type 1))
- (seq (dependencies.db/deleted-entity-ids entity-type 1))))
+ (or (dependencies.db/dependency-pending?
+ entity-type models.dependency/current-dependency-analysis-version)
+ (seq (dependencies.db/dependency-deleted-entity-ids entity-type 1))))
deps.dependency-types/backfillable-dependency-types)))
-
-(defn record-failure!
- "Record a failed dependency calculation attempt for an entity.
- Increments fail_count and sets next_retry_at based on exponential backoff.
- If max retries exceeded, marks the entity as terminal.
- Creates the entry if it doesn't exist, since entities with no row yet are exactly the ones
- [[instances-for-dependency-calculation]] picks up first."
- [entity-type entity-id max-retries delay-minutes]
- (dependencies.db/record-dependency-status-failure!
- entity-type entity-id
- (fn [existing]
- ;; An inserted row takes the column defaults for `stale` (false) and `dependency_analysis_version` (0). 0 is
- ;; below `current-dependency-analysis-version`, which is what keeps the entity eligible for the retry once
- ;; `next_retry_at` has elapsed.
- (let [new-fail-count (inc (:fail_count existing 0))]
- (if (> new-fail-count max-retries)
- {:fail_count new-fail-count
- :terminal true
- :next_retry_at nil}
- (let [retry-minutes (* new-fail-count delay-minutes)]
- {:fail_count new-fail-count
- :next_retry_at (when (pos? retry-minutes)
- (t/plus (t/offset-date-time) (t/minutes retry-minutes)))}))))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.clj
new file mode 100644
index 00000000000..2c3e5bdd8e5
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.clj
@@ -0,0 +1,21 @@
+(ns metabase-enterprise.dependencies.models.result-metadata-status
+ "The ResultMetadataStatus model: one status per MBQL card, marked stale by the writes its result metadata reads."
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/ResultMetadataStatus [_model] :result_metadata_status)
+
+(derive :model/ResultMetadataStatus :metabase/model)
+
+(t2/deftransforms :model/ResultMetadataStatus
+ {:entity_type mi/transform-keyword})
+
+(def current-result-metadata-version
+ "Current version of the result metadata logic. Increment it when the logic changes, so the job processes every
+ entity again.
+
+ Version history:
+ - 1: Initial version"
+ 1)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.clj b/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.clj
new file mode 100644
index 00000000000..d6a2c2690ad
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.clj
@@ -0,0 +1,160 @@
+(ns metabase-enterprise.dependencies.result-metadata
+ "Re-derives the result metadata of the MBQL cards whose ResultMetadataStatus a write marked stale, which a write of
+ what they read does; native cards are left alone, since their result metadata comes from running them."
+ (:require
+ [clojure.walk :as walk]
+ [malli.error :as me]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.result-metadata-status :as models.result-metadata-status]
+ [metabase-enterprise.dependencies.schema :as dependencies.schema]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.queries.core :as queries]
+ [metabase.types.core :as types]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private compared-column-keys
+ "The result metadata column keys that the queries using a card as a source see."
+ [:name :display_name :description :semantic_type :fk_target_field_id :id :settings :visibility_type
+ :base_type :effective_type :unit :inherited_temporal_unit])
+
+(def ^:private measured-column-keys
+ "The result metadata column keys a run of the card measures, within the type that inference gets right."
+ [:base_type :effective_type :database_type])
+
+(def ^:private Column
+ "A result metadata column, as a card stores or inference returns it, valid or not."
+ [:fn {:error/message "a map"} map?])
+
+(def ^:private TypeName
+ "A type, as a keyword or as the string a stored column may hold it as."
+ [:maybe [:or :keyword :string]])
+
+(mu/defn- normalize-column :- Column
+ "The result metadata column `col` with its unqualified keys in snake_case."
+ [col :- Column]
+ (update-keys col #(cond-> % (not (qualified-keyword? %)) u/->snake_case_en)))
+
+(mu/defn- comparable-metadata :- [:vector [:map-of :keyword :any]]
+ "The [[compared-column-keys]] of the result metadata `cols` with a value, with keyword values as strings."
+ [cols :- [:sequential Column]]
+ (mapv (fn [col]
+ (into {}
+ (keep (fn [k]
+ (when-some [v (get col k)]
+ [k (walk/postwalk #(cond-> % (keyword? %) u/qualified-name) v)])))
+ compared-column-keys))
+ cols))
+
+(mu/defn same-output? :- :boolean
+ "Whether the result metadata `before` and `after` show the queries using the card the same columns."
+ [before :- [:maybe [:sequential Column]]
+ after :- [:maybe [:sequential Column]]]
+ (= (comparable-metadata (map normalize-column before))
+ (comparable-metadata (map normalize-column after))))
+
+(mu/defn- same-root-type? :- [:maybe :boolean]
+ "Whether the types `x` and `y` share an ancestor below `:type/*`, like integers and decimals do."
+ [x :- TypeName
+ y :- TypeName]
+ (and x y (not= :type/* (types/most-specific-common-ancestor (keyword x) (keyword y)))))
+
+(mu/defn- measured-by-a-run? :- [:maybe :boolean]
+ "Whether the stored column `stored-col` holds what a run measured for the inferred column `col`: a run stored its
+ source type, they have types of the same root, and the column's source type is unchanged or unknown."
+ [stored-col :- Column
+ col :- Column]
+ (and (some? (:database_type stored-col))
+ (same-root-type? (:base_type stored-col) (:base_type col))
+ (or (nil? (:database_type col))
+ (= (:database_type col) (:database_type stored-col)))))
+
+(mu/defn- patch-result-metadata :- [:vector Column]
+ "The inferred result metadata `inferred`, each column keeping the [[measured-column-keys]] of the stored column of
+ `stored` it stands for when a run measured them, and only that column's fingerprint, so that the job never shows a
+ fingerprint the card's runs and saves did not store."
+ [stored :- [:sequential Column]
+ inferred :- [:sequential Column]]
+ (mapv (fn [col stored-col]
+ (cond-> (dissoc col :fingerprint)
+ (and stored-col (measured-by-a-run? stored-col col))
+ (merge (select-keys stored-col measured-column-keys))
+
+ (some? (:fingerprint stored-col))
+ (assoc :fingerprint (:fingerprint stored-col))))
+ inferred
+ (lib/match-returned-columns stored inferred)))
+
+(mu/defn- rederived-result-metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]
+ "The re-derived result metadata of the MBQL `card`, inferred as no user so no one's permissions decide it, or nil
+ when what the queries using it see is unchanged."
+ [{query :dataset_query, stored :result_metadata, :keys [id], :as card} :- (ms/InstanceOf :model/Card)]
+ (let [stored (mapv normalize-column stored)
+ metadata (patch-result-metadata
+ stored
+ (or (queries/infer-metadata-with-model-overrides query card)
+ (throw (ex-info "Could not infer the result metadata of the card" {:card-id id}))))]
+ (when (not= (comparable-metadata stored) (comparable-metadata metadata))
+ (when-let [error (me/humanize (mr/explain ::lib.schema.metadata/card.result-metadata metadata))]
+ (throw (ex-info "Invalid re-derived result metadata" {:card-id id :error error})))
+ metadata)))
+
+(mu/defn- store-result-metadata!
+ "Mark the entities directly depending on the Card with `card-id` stale, then, under the card's row lock, store its
+ re-derived result metadata `metadata` through the Card model when the card is still in the `state` it was re-derived
+ from, or else mark the card stale, so it is re-derived from what a save or a run stored meanwhile."
+ [card-id :- ::lib.schema.id/card
+ state :- ::dependencies.schema/card-state
+ metadata :- ::lib.schema.metadata/card.result-metadata]
+ (t2/with-transaction [_]
+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! :card #{card-id})
+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! :card #{card-id})
+ (if (= state (dependencies.db/card-state card-id true))
+ (dependencies.db/set-card-result-metadata! card-id metadata)
+ (dependencies.db/mark-result-metadata-statuses-stale! {:card #{card-id}}))))
+
+(mu/defn- rederive-card!
+ "Re-derive the result metadata of the MBQL Card with `card-id` and store it when it changed."
+ [card-id :- ::lib.schema.id/card]
+ (let [state (dependencies.db/card-state card-id false)
+ card (dependencies.db/card card-id)
+ metadata (lib-be/with-metadata-provider-cache
+ (when (and card (not= :native (:query_type card)) (seq (:dataset_query card)))
+ (rederived-result-metadata card)))]
+ (when (and state metadata)
+ (store-result-metadata! card-id state metadata))))
+
+(mu/defn update-result-metadata! :- nat-int?
+ "Delete the statuses of deleted cards, then re-derive up to `batch-size` MBQL cards whose ResultMetadataStatus counts
+ a write, is missing or is outdated, clearing each one's `stale` count unless a write raised it meanwhile, or counting
+ its failure, returning how many statuses it took, deleted ones included."
+ [batch-size :- ms/PositiveInt]
+ (let [deleted-ids (dependencies.db/result-metadata-deleted-card-ids batch-size)
+ statuses (dependencies.db/take-pending-result-metadata-statuses!
+ batch-size models.result-metadata-status/current-result-metadata-version
+ models.dependency/current-dependency-analysis-version)]
+ (when (seq deleted-ids)
+ (dependencies.db/delete-result-metadata-statuses! deleted-ids))
+ (doseq [{card-id :entity_id, :keys [stale]} statuses]
+ (try
+ (rederive-card! card-id)
+ (dependencies.db/clear-result-metadata-status!
+ card-id stale models.result-metadata-status/current-result-metadata-version)
+ (catch Throwable e
+ (log/warnf "Failed to re-derive the result metadata of card %s: %s" card-id (ex-message e))
+ (when (dependencies.db/record-result-metadata-failure! card-id (deps.settings/dependency-max-failures))
+ (log/warnf "Giving up on the result metadata of card %s until it changes" card-id))
+ (when-not (instance? Exception e)
+ (throw e)))))
+ (+ (count deleted-ids) (count statuses))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
index fe54439a1d9..71a29b23408 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
@@ -2,9 +2,61 @@
(:require
[metabase.documents.schema :as documents.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.schema :as queries.schema]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
+(mr/def ::entity-type
+ "A dependency-type keyword, or the equivalent string, which the keyword transform of every entity type column coerces."
+ [:or :metabase-enterprise.dependencies.dependency-types/dependency-types :string])
+
+(mr/def ::node
+ "A dependency graph node: an entity as `[entity-type entity-id]`."
+ [:tuple
+ :metabase-enterprise.dependencies.dependency-types/dependency-types
+ :metabase-enterprise.dependencies.dependency-types/entity-id])
+
+(mr/def ::child-map
+ "The children of dependency graph nodes, as `{node #{child-node}}`."
+ [:map-of ::node [:set ::node]])
+
+(mr/def ::visible-opts
+ "The user whose readable entities a query is restricted to (`:user-id`, `:is-superuser?`, `:is-data-analyst?`), and
+ whether archived entities count (`:include-archived-items`, default `:exclude`)."
+ [:map {:closed true}
+ [:user-id ::lib.schema.id/user]
+ [:is-superuser? {:optional true} [:maybe :boolean]]
+ [:is-data-analyst? {:optional true} [:maybe :boolean]]
+ [:include-archived-items {:optional true} [:enum :exclude :all :only]]])
+
+(mr/def ::edge-restriction
+ "The restriction of one side of a dependency edge, nil for none: the entities readable by a user (`:visible`), the
+ entities that failed analysis (`:broken?`), the entities of some types (`:types`), and the entities of one type with
+ some ids (`:entity-type` and `:ids`)."
+ [:maybe
+ [:map {:closed true}
+ [:visible {:optional true} ::visible-opts]
+ [:broken? {:optional true} :boolean]
+ [:types {:optional true} [:set ::entity-type]]
+ [:entity-type {:optional true} ::entity-type]
+ [:ids {:optional true} [:sequential :metabase-enterprise.dependencies.dependency-types/entity-id]]]])
+
+(mr/def ::pending-status
+ "An entity a job is to process, with the `stale` count of its status when the job picked it, nil without a status."
+ [:map {:closed true}
+ [:entity_id ms/PositiveInt]
+ [:stale [:maybe ms/IntGreaterThanOrEqualToZero]]])
+
+(mr/def ::type->ids
+ "Entities as `{entity-type ids}`."
+ [:map-of :metabase-enterprise.dependencies.dependency-types/dependency-types [:set ms/PositiveInt]])
+
+(mr/def ::card-state
+ "The `updated_at` and `result_metadata` of a Card, which a save or a run changes."
+ [:map {:closed true}
+ [:updated_at ms/TemporalInstant]
+ [:result_metadata [:maybe ::queries.schema/card.result-metadata]]])
+
(mr/def ::upstream-deps
[:map
[:card {:optional true} [:set ::lib.schema.id/card]]
@@ -14,27 +66,7 @@
[:dashboard {:optional true} [:set ::lib.schema.id/dashboard]]
[:document {:optional true} [:set ::documents.schema/document.id]]
[:sandbox {:optional true} [:set ::lib.schema.id/sandbox]]
- [:segment {:optional true} [:set ::lib.schema.id/segment]]
- [:unresolved-table {:optional true} [:set [:map
- [:schema {:optional true} :string]
- [:table :string]]]]])
-
-(mr/def ::analysis-finding
- "A AnalysisFinding as selected from the app DB: every column of `:analysis_finding`."
- [:merge
- ::analysis-finding.update
- [:map {:closed true}
- [:id ms/PositiveInt]]])
-
-(mr/def ::analysis-finding.update
- "What an update (or insert) of a AnalysisFinding accepts: every column of `:analysis_finding` except `id`, all optional."
- [:map {:closed true}
- [:analyzed_entity_type {:optional true} [:maybe [:or :keyword :string]]]
- [:analyzed_entity_id {:optional true} [:maybe ms/PositiveInt]]
- [:analysis_version {:optional true} [:maybe :int]]
- [:analyzed_at {:optional true} [:maybe ms/TemporalInstant]]
- [:result {:optional true} [:maybe :boolean]]
- [:stale {:optional true} [:maybe :boolean]]])
+ [:segment {:optional true} [:set ::lib.schema.id/segment]]])
(mr/def ::analysis-finding-error
"A AnalysisFindingError as selected from the app DB: every column of `:analysis_finding_error`."
@@ -67,22 +99,3 @@
[:from_entity_id {:optional true} [:maybe ms/PositiveInt]]
[:to_entity_type {:optional true} [:maybe [:or :keyword :string]]]
[:to_entity_id {:optional true} [:maybe ms/PositiveInt]]])
-
-(mr/def ::dependency-status
- "A DependencyStatus as selected from the app DB: every column of `:dependency_status`."
- [:merge
- ::dependency-status.update
- [:map {:closed true}
- [:id ms/PositiveInt]]])
-
-(mr/def ::dependency-status.update
- "What an update (or insert) of a DependencyStatus accepts: every column of `:dependency_status` except `id`, all optional."
- [:map {:closed true}
- [:entity_type {:optional true} [:maybe [:or :keyword :string]]]
- [:entity_id {:optional true} [:maybe ms/PositiveInt]]
- [:dependency_analysis_version {:optional true} [:maybe :int]]
- [:stale {:optional true} [:maybe :boolean]]
- [:fail_count {:optional true} [:maybe :int]]
- [:next_retry_at {:optional true} [:maybe ms/TemporalInstant]]
- [:terminal {:optional true} [:maybe :boolean]]
- [:unresolved_tables {:optional true} [:maybe :boolean]]])
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj
index c66e6b0eaa4..6bd36a7722d 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj
@@ -3,7 +3,7 @@
[metabase.settings.core :refer [defsetting]]))
(defsetting dependency-backfill-batch-size
- "The number of entities that the dependency backfill job will process per run."
+ "The number of entities that the dependency job will process per run."
:visibility :internal
:type :integer
:default 500
@@ -11,7 +11,7 @@
:export? false)
(defsetting dependency-backfill-delay-minutes
- "The average delay between when the dependency backfill job will run."
+ "The average delay before the dependency job runs again after a run that failed."
:visibility :internal
:type :integer
:default 60
@@ -19,9 +19,8 @@
:export? false)
(defsetting dependency-backfill-variance-minutes
- "The variation in when the dependency backfill job will run. The actual delay between runs will be a random value
- between `dependency-backfill-delay-minutes - dependency-backfill-variance-minutes` and
- `dependency-backfill-delay-minutes + dependency-backfill-variance-minutes`."
+ "The variation of the dependency job's delay after a failed run, which is a random value within this many minutes of
+ `dependency-backfill-delay-minutes`, and the longest random delay of its first run after startup."
:visibility :internal
:type :integer
:default 30
@@ -29,7 +28,7 @@
:export? false)
(defsetting dependency-entity-check-batch-size
- "The number of entities that the entity check job will process per run."
+ "The number of entities that the analysis finding job will process per run."
:visibility :internal
:type :integer
:default 500
@@ -37,7 +36,7 @@
:export? false)
(defsetting dependency-entity-check-delay-minutes
- "The average delay between when the entity check job will run."
+ "The average delay before the analysis finding job runs again after a run that failed."
:visibility :internal
:type :integer
:default 60
@@ -45,11 +44,44 @@
:export? false)
(defsetting dependency-entity-check-variance-minutes
- "The variation in when the entity check job will run. The actual delay between runs will be a random value
- between `dependency-entity-check-delay-minutes - dependency-entity-check-variance-minutes` and
- `dependency-entity-check-delay-minutes + dependency-entity-check-variance-minutes`."
+ "The variation of the analysis finding job's delay after a failed run, which is a random value within this many
+ minutes of `dependency-entity-check-delay-minutes`, and the longest random delay of its first run after startup."
:visibility :internal
:type :integer
:default 30
:setter :none
:export? false)
+
+(defsetting dependency-result-metadata-batch-size
+ "The number of entities that the result metadata job will process per run."
+ :visibility :internal
+ :type :integer
+ :default 500
+ :setter :none
+ :export? false)
+
+(defsetting dependency-result-metadata-delay-minutes
+ "The average delay before the result metadata job runs again after a run that failed."
+ :visibility :internal
+ :type :integer
+ :default 60
+ :setter :none
+ :export? false)
+
+(defsetting dependency-result-metadata-variance-minutes
+ "The variation of the result metadata job's delay after a failed run, which is a random value within this many
+ minutes of `dependency-result-metadata-delay-minutes`, and the longest random delay of its first run after startup."
+ :visibility :internal
+ :type :integer
+ :default 30
+ :setter :none
+ :export? false)
+
+(defsetting dependency-max-failures
+ "How many times in a row a dependency job fails to process an entity before it gives up on it until a write changes
+ it."
+ :visibility :internal
+ :type :integer
+ :default 3
+ :setter :none
+ :export? false)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.clj
new file mode 100644
index 00000000000..3022b83d074
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.clj
@@ -0,0 +1,53 @@
+(ns metabase-enterprise.dependencies.task.analysis-finding
+ "The job that runs the analysis of the entities whose AnalysisFindingStatus is stale, missing or outdated."
+ (:require
+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase-enterprise.dependencies.task.util :as task.util]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn- run-batch! :- [:maybe ms/PositiveInt]
+ "Analyze a batch of entities whose findings are outdated, returning when to run again: a second from now when it took
+ any, or nil."
+ []
+ (when (premium-features/has-feature? :dependencies)
+ (let [batch-size (deps.settings/dependency-entity-check-batch-size)
+ taken (deps.analysis-finding/analyze-batch! batch-size)]
+ (when (pos? taken)
+ (log/debug "Processed" taken "entities for analysis")
+ 1))))
+
+(declare ^:private job)
+
+(task/defjob UpdateAnalysisFindings
+ "Re-analyze the entities that changed or whose upstreams changed."
+ {:saved-class "metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"
+ :concurrent? false}
+ [ctx]
+ (log/debug "Executing UpdateAnalysisFindings job...")
+ (task.util/run-and-reschedule! (.getScheduler ctx) job))
+
+(def ^:private job
+ {:job-type UpdateAnalysisFindings
+ :job-key "metabase.dependencies.task.entity-check.job"
+ :trigger-key "metabase.dependencies.task.entity-check.trigger"
+ :job-name "Update Analysis Findings"
+ :batch-size deps.settings/dependency-entity-check-batch-size
+ :delay-minutes deps.settings/dependency-entity-check-delay-minutes
+ :variance-minutes deps.settings/dependency-entity-check-variance-minutes
+ :run-batch! #'run-batch!})
+
+(defmethod task/init! ::UpdateAnalysisFindings [_]
+ (task.util/init-job! job))
+
+(mu/defn trigger-analysis-finding-job! :- :any
+ "Trigger the UpdateAnalysisFindings job, unless a MB_DEPENDENCY_ENTITY_CHECK_BATCH_SIZE that isn't positive
+ disables it."
+ []
+ (task.util/trigger-run! job))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
deleted file mode 100644
index cda348a3988..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
+++ /dev/null
@@ -1,268 +0,0 @@
-(ns metabase-enterprise.dependencies.task.backfill
- "Implements a task that brings all entities with dependencies tracked in the dependency table
- (see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure
- the dependency table contains fresh entries.
-
- This is done by querying the dependency_status table for entities that are stale or have an
- outdated dependency_analysis_version. The backfill task computes dependencies and updates
- the dependency_status table. Entities that a write marked stale, or that were deleted, also have their dependents
- marked for re-analysis here, outside the write's transaction."
- (:require
- [clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.scheduler :as qs]
- [clojurewerkz.quartzite.triggers :as triggers]
- [java-time.api :as t]
- [metabase-enterprise.dependencies.calculation :as deps.calculation]
- [metabase-enterprise.dependencies.db :as dependencies.db]
- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.settings :as deps.settings]
- [metabase-enterprise.dependencies.task-util :as deps.task-util]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
- [metabase.graph.core :as graph]
- [metabase.lib-be.core :as lib-be]
- [metabase.premium-features.core :as premium-features]
- [metabase.task.core :as task]
- [metabase.transforms.core :as transforms]
- [metabase.util.log :as log]
- [toucan2.core :as t2])
- (:import (org.quartz JobExecutionContext)))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private expanded-into-dependents
- "The entity types whose content a native query using them expands into its own SQL, so that a change to one changes
- the table dependencies of the entities using it, which no graph edge carries."
- #{:snippet})
-
-(def ^:private entity-types
- "The list of entity types to backfill, those expanded into their dependents first, so that the dependents they mark
- are recomputed in the same run.
-
- This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, links
- involving tables are found via analysis of the other side of the relation."
- (vec (sort-by (complement expanded-into-dependents) deps.dependency-types/backfillable-dependency-types)))
-
-(def ^:private max-retries 5)
-
-;;; ------------------------------ Post-deps cleanup multimethod ------------------------------
-
-(defmulti post-deps-cleanup!
- "Perform entity-specific cleanup after dependencies have been replaced.
- For example, transforms need to clean up outdated downstream table->transform dependencies.
- Default is no-op."
- {:arglists '([entity-type entity])}
- (fn [entity-type _entity] entity-type))
-
-(defmethod post-deps-cleanup! :default [_ _entity] nil)
-
-(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]
- (let [db-id (transforms/transform-source-database transform)
- downstream-table-ids (dependencies.db/downstream-table-ids-of-transform id)
- downstream-tables (when (seq downstream-table-ids)
- (dependencies.db/tables downstream-table-ids))
- outdated-tables (remove (fn [table]
- (and (= (:schema table) (:schema target))
- (= (:name table) (:name target))
- (or (not db-id)
- (= db-id (:db_id table)))))
- downstream-tables)
- not-found-table-ids (remove (into #{} (map :id) downstream-tables)
- downstream-table-ids)]
- (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
- (map :id) outdated-tables))]
- (dependencies.db/delete-table-dependencies-on-transform! outdated-downstream-table-ids id))))
-
-;;; ------------------------------ Backfill orchestration ------------------------------
-
-(defn- processable-instances [entity-type batch-size]
- (deps.dependency-status/hydrate-for-deps
- entity-type
- (deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))
-
-(defn- mark-expanding-entities-stale!
- "Mark for recalculation the entities whose native SQL expands the `entity-type` entity `entity-id`, directly or
- through the entities of the same kind that include it."
- [entity-type entity-id]
- (let [graph (models.dependency/filtered-graph-dependents nil {:types expanded-into-dependents})
- type->ids (models.dependency/group-nodes (graph/transitive graph [[entity-type entity-id]]))]
- (deps.dependency-status/mark-stale! (apply dissoc type->ids expanded-into-dependents))))
-
-(defn- compute-deps-for-entity!
- "Mark the dependents of an entity that a write marked stale for re-analysis, and those that expand it for
- recalculation, then compute and store its dependencies and update its dependency_status, flagging the tables it
- names that match nothing. Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."
- [entity-type entity]
- (log/debug "Computing dependencies for" (name entity-type) (:id entity))
- (when (:stale (dependencies.db/dependency-status entity-type (:id entity)))
- (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}})
- (when (expanded-into-dependents entity-type)
- (mark-expanding-entities-stale! entity-type (:id entity))))
- (t2/with-transaction [_]
- (let [deps (deps.calculation/calculate-deps entity-type entity)]
- (models.dependency/replace-dependencies! entity-type (:id entity) (dissoc deps :unresolved-table))
- (post-deps-cleanup! entity-type entity)
- (deps.dependency-status/upsert-status! entity-type (:id entity) (contains? deps :unresolved-table)))))
-
-(defn- remove-deleted-entity-batch!
- "Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop their
- dependencies and status, returning how many were removed."
- [entity-type batch-size]
- (let [entity-ids (dependencies.db/deleted-entity-ids entity-type batch-size)]
- (when (seq entity-ids)
- (t2/with-transaction [_]
- (deps.findings/mark-transitive-dependents-stale! {entity-type entity-ids})
- (dependencies.db/delete-dependencies-from! entity-type entity-ids)
- (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))
- (count entity-ids)))
-
-(defn- backfill-entity-batch!
- [entity-type batch-size]
- (lib-be/with-metadata-provider-cache
- (let [type-name (name entity-type)
- instances (processable-instances entity-type batch-size)]
- (when (seq instances)
- (log/infof "Processing a batch of %s %s(s)..." (count instances) type-name))
- (reduce (fn [total entity]
- (+ total
- (try
- (compute-deps-for-entity! entity-type entity)
- 1
- (catch Throwable e ;; catch OOMs
- (let [id (:id entity)]
- (try
- (deps.dependency-status/record-failure!
- entity-type id max-retries
- (deps.settings/dependency-backfill-delay-minutes))
- (let [{:keys [fail_count terminal]} (dependencies.db/dependency-status entity-type id)]
- (if terminal
- (log/errorf "Entity %s %s failed %d times, marking as terminally broken: %s"
- type-name id fail_count (ex-message e))
- (log/warnf "Entity %s %s failed, failure count: %d. %s"
- type-name id fail_count (ex-message e))))
- (catch Exception record-ex
- (log/errorf "Entity %s %s failed during dependency calculation: %s"
- type-name id (ex-message e))
- (log/errorf "Additionally, failed to record the failure for %s %s: %s"
- type-name id (ex-message record-ex)))))
- (when-not (instance? Exception e) ;; re-throw Errors
- (throw e))
- 0))))
- 0
- instances))))
-
-(defn- backfill-dependencies!
- "Job to backfill dependencies for all entities, triggering the entity-check job for the findings it marked stale.
- Returns true if a full batch has been selected, nil or false otherwise."
- []
- (when (premium-features/has-feature? :dependencies)
- (let [batch-size (deps.settings/dependency-backfill-batch-size)
- remaining (reduce (fn [remaining entity-type]
- (if (< remaining 1)
- (reduced 0)
- (let [removed (remove-deleted-entity-batch! entity-type remaining)
- processed (if (< removed remaining)
- (backfill-entity-batch! entity-type (- remaining removed))
- 0)]
- (when (pos? processed)
- (log/info "Updated" processed "entities."))
- (- remaining removed processed))))
- batch-size
- entity-types)]
- (when (< remaining batch-size)
- (task.entity-check/trigger-entity-check-job!))
- (< remaining 1))))
-
-(defn- has-pending-retries?
- "Whether any entity is in retry backoff that this instance can actually act on.
-
- Fails open on an indeterminate token status. `has-feature?` cannot tell a network failure from being unlicensed, and
- both inputs to the job's reschedule decision consult the licence, so reading a blip as `false` would stop the job
- for good — nothing re-fires when the check recovers, because the token never changed. Only a definitive `false`
- counts as unlicensed here — which is the common case, since no token at all answers `false`. A token that cannot be
- validated answers `nil` and keeps the job alive. Doing the work still fails closed — see
- [[backfill-dependencies!]]."
- []
- (and (not (false? (premium-features/canonically-has-feature? :dependencies)))
- (deps.dependency-status/has-pending-retries?)))
-
-(declare schedule-run!)
-
-(defn- next-run-delay []
- (deps.task-util/job-delay
- (deps.settings/dependency-backfill-delay-minutes)
- (deps.settings/dependency-backfill-variance-minutes)))
-
-(defn- run-and-reschedule!
- "Process a batch, then schedule the next run if there is more to do.
-
- Reschedules on the way out of a failure too, then rethrows. This job is one-shot self-rescheduling with no cron
- backstop, so anything escaping the batch would otherwise end the chain until a content-change event or a restart —
- survivable for an `OutOfMemoryError` that takes the process with it, not for a `StackOverflowError` from
- pathological SQL that leaves a live process behind."
- [scheduler]
- (try
- (let [full-batch-selected? (backfill-dependencies!)
- retries? (has-pending-retries?)]
- (if (or full-batch-selected? retries?)
- (schedule-run! scheduler (next-run-delay))
- (log/info "No more entities to backfill for, stopping.")))
- (catch Throwable e
- (schedule-run! scheduler (next-run-delay))
- (throw e))))
-
-(defn- log-job-start
- [^JobExecutionContext ctx]
- (let [scheduler (.getScheduler ctx)
- job-key (.getKey (.getJobDetail ctx))
- job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))
- active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]
- (log/infof "Executing %s job. %s total triggers active." job-class active-triggers-count)))
-
-(task/defjob BackfillDependencies
- "Backfill the dependency table."
- {:saved-class "metabase_enterprise.dependencies.task.backfill.BackfillDependencies"
- :concurrent? false}
- [ctx]
- (let [ctx ^JobExecutionContext ctx]
- (log-job-start ctx)
- (run-and-reschedule! (.getScheduler ctx))))
-
-(def ^:private job-key "metabase.task.dependency-backfill.job")
-(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")
-
-(defn- schedule-run!
- "Schedule a run of the backfill job `delay-in-seconds` from now."
- [scheduler delay-in-seconds]
- (let [start-at (-> (t/instant)
- (t/+ (t/duration delay-in-seconds :seconds))
- java.util.Date/from)
- trigger (triggers/build
- (triggers/with-identity (triggers/key trigger-key))
- (triggers/for-job job-key)
- (triggers/start-at start-at))
- job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]
- (log/info "Scheduling next run of job Dependency Backfill at" start-at)
- (task/schedule-task! scheduler job trigger)))
-
-(defn trigger-backfill-job!
- "Trigger the BackfillDependencies job to run after a brief delay, unless the batch size disables it.
-
- The 1-second delay ensures the calling transaction has committed before the job checks for stale entities. Entity
- changes fire this, so leaving it ungated is what made a disabled job wake roughly once a second; the job's own
- periodic schedule is left alone so it still resumes if the batch size becomes positive.
-
- Disable with MB_DEPENDENCY_BACKFILL_BATCH_SIZE=0"
- []
- (when (pos? (deps.settings/dependency-backfill-batch-size))
- (schedule-run! (task/scheduler) 1)))
-
-(defmethod task/init! ::DependencyBackfill [_]
- (when-not (pos? (deps.settings/dependency-backfill-batch-size))
- (log/info "Dependency backfill batch size is not positive; the job will run but process nothing"))
- (schedule-run!
- (task/scheduler)
- (deps.task-util/job-initial-delay
- (deps.settings/dependency-backfill-variance-minutes))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.clj
new file mode 100644
index 00000000000..f5219d4843d
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.clj
@@ -0,0 +1,200 @@
+(ns metabase-enterprise.dependencies.task.dependency
+ "The job that computes the dependencies of the entities whose dependency_status counts a write, records an older
+ analysis_version or is missing, clearing each count afterwards unless a write raised it meanwhile."
+ (:require
+ [clojurewerkz.quartzite.scheduler :as qs]
+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]
+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]
+ [metabase-enterprise.dependencies.task.util :as task.util]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.task.core :as task]
+ [metabase.transforms.core :as transforms]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2])
+ (:import (org.quartz JobExecutionContext)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private entity-types
+ "The list of entity types whose dependencies the job computes, snippets first, so that the snippets the entities
+ using them expand are known when those are computed.
+
+ This is not the same as deps.dependency-types/dependency-types, because tables have no dependencies of their
+ own. Instead, links involving tables are found via analysis of the other side of the relation."
+ (vec (sort-by #(not= :snippet %) deps.dependency-types/backfillable-dependency-types)))
+
+(def ^:private Entity
+ "An entity whose dependencies the job computes, as the app DB holds it."
+ [:fn {:error/message "a Toucan entity"} t2/instance?])
+
+;;; ------------------------------ Post-deps cleanup multimethod ------------------------------
+
+(defmulti post-deps-cleanup!
+ "Perform entity-specific cleanup after dependencies have been replaced.
+ For example, transforms need to clean up outdated downstream table->transform dependencies.
+ Default is no-op."
+ {:arglists '([entity-type entity])}
+ (fn [entity-type _entity] entity-type))
+
+(defmethod post-deps-cleanup! :default [_ _entity] nil)
+
+(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]
+ (let [db-id (transforms/transform-source-database transform)
+ downstream-table-ids (dependencies.db/downstream-table-ids-of-transform id)
+ downstream-tables (when (seq downstream-table-ids)
+ (dependencies.db/tables downstream-table-ids))
+ outdated-tables (remove (fn [table]
+ (and (= (:schema table) (:schema target))
+ (= (:name table) (:name target))
+ (or (not db-id)
+ (= db-id (:db_id table)))))
+ downstream-tables)
+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)
+ downstream-table-ids)]
+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
+ (map :id) outdated-tables))]
+ (dependencies.db/delete-table-dependencies-on-transform! outdated-downstream-table-ids id))))
+
+;;; ------------------------------ Job orchestration ------------------------------
+
+(mu/defn- hydrated-entities :- [:sequential Entity]
+ "The entities of `entity-type` with `entity-ids`, hydrated for computing their dependencies."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity-ids :- [:set ms/PositiveInt]]
+ (vec (deps.dependency-status/hydrate-for-deps
+ entity-type (vec (dependencies.db/entities entity-type entity-ids)))))
+
+(mu/defn- compute-deps-for-entity! :- :any
+ "Compute and store the dependencies of an `entity` hydrated by [[deps.dependency-status/hydrate-for-deps]], marking
+ an entity with a new upstream stale for its result metadata and analysis, or for a snippet the analysis of what
+ expands it, which an upstream change made before the edge existed never marked."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ entity :- Entity]
+ (log/debug "Computing dependencies for" (name entity-type) (:id entity))
+ (let [deps (deps.calculation/calculate-deps entity-type entity)
+ own {entity-type #{(:id entity)}}]
+ (t2/with-transaction [_]
+ (when (seq (models.dependency/new-dependencies entity-type (:id entity) deps))
+ (when (= :card entity-type)
+ (dependencies.db/mark-result-metadata-statuses-stale! own))
+ (when (contains? deps.analysis-finding/analyzable-entities entity-type)
+ (dependencies.db/mark-analysis-finding-statuses-stale! own))
+ (when (= :snippet entity-type)
+ (dependencies.db/mark-analysis-finding-statuses-stale!
+ (models.dependency/transitive-dependents {:snippet [entity]}))))
+ (models.dependency/replace-dependencies! entity-type (:id entity) deps)
+ (post-deps-cleanup! entity-type entity))))
+
+(mu/defn- remove-deleted-entity-batch! :- ms/IntGreaterThanOrEqualToZero
+ "Drop the dependencies and status of up to `batch-size` deleted entities of `entity-type`, returning how many were
+ removed."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (let [entity-ids (dependencies.db/dependency-deleted-entity-ids entity-type batch-size)]
+ (when (seq entity-ids)
+ (t2/with-transaction [_]
+ (dependencies.db/delete-dependencies-from! entity-type entity-ids)
+ (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))
+ (count entity-ids)))
+
+(mu/defn- update-entity-batch! :- ms/IntGreaterThanOrEqualToZero
+ "Compute the dependencies of up to `batch-size` pending entities of `entity-type`, clearing each one's `stale` count
+ unless a write raised it meanwhile, or counting its failure, returning how many it took."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (lib-be/with-metadata-provider-cache
+ (let [type-name (name entity-type)
+ statuses (deps.dependency-status/take-pending-statuses! entity-type batch-size)
+ stale (into {} (map (juxt :entity_id :stale)) statuses)]
+ (when (seq statuses)
+ (log/debugf "Processing a batch of %s %s(s)..." (count statuses) type-name))
+ (doseq [entity (when (seq statuses) (hydrated-entities entity-type (set (keys stale))))]
+ (try
+ (compute-deps-for-entity! entity-type entity)
+ (deps.dependency-status/clear-status! entity-type (:id entity) (stale (:id entity)))
+ (catch Throwable e ;; catch OOMs
+ (log/errorf "Computing the dependencies of %s %s failed: %s" type-name (:id entity) (ex-message e))
+ (when (dependencies.db/record-dependency-failure! entity-type (:id entity)
+ (deps.settings/dependency-max-failures))
+ (log/warnf "Giving up on the dependencies of %s %s until it changes" type-name (:id entity)))
+ (when-not (instance? Exception e) ;; re-throw Errors
+ (throw e)))))
+ (count statuses))))
+
+(mu/defn- update-dependencies! :- [:maybe :boolean]
+ "Update the dependencies of up to a batch of entities, triggering the jobs that read them, and return whether the
+ batch took any."
+ []
+ (when (premium-features/has-feature? :dependencies)
+ (let [batch-size (deps.settings/dependency-backfill-batch-size)
+ remaining (reduce (fn [remaining entity-type]
+ (if (< remaining 1)
+ (reduced 0)
+ (let [removed (remove-deleted-entity-batch! entity-type remaining)
+ processed (if (< removed remaining)
+ (update-entity-batch! entity-type (- remaining removed))
+ 0)]
+ (when (pos? processed)
+ (log/debug "Updated" processed "entities."))
+ (- remaining removed processed))))
+ batch-size
+ entity-types)]
+ (when (< remaining batch-size)
+ (task.analysis-finding/trigger-analysis-finding-job!)
+ (task.result-metadata/trigger-result-metadata-job!)
+ true))))
+
+(declare ^:private job)
+
+(mu/defn- run-batch! :- [:maybe ms/PositiveInt]
+ "Process a batch, returning when to run again: a second from now when it took any, or nil."
+ []
+ (when (update-dependencies!)
+ 1))
+
+(mu/defn- log-job-start :- :nil
+ "Log the start of the job running in `ctx`, with how many triggers it has."
+ [^JobExecutionContext ctx :- (ms/InstanceOfClass JobExecutionContext)]
+ (let [scheduler (.getScheduler ctx)
+ job-key (.getKey (.getJobDetail ctx))
+ job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))
+ active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]
+ (log/debugf "Executing %s job. %s total triggers active." job-class active-triggers-count)))
+
+(task/defjob UpdateDependencies
+ "Update the dependency table."
+ {:saved-class "metabase_enterprise.dependencies.task.backfill.BackfillDependencies"
+ :concurrent? false}
+ [ctx]
+ (let [ctx ^JobExecutionContext ctx]
+ (log-job-start ctx)
+ (task.util/run-and-reschedule! (.getScheduler ctx) job)))
+
+(def ^:private job
+ {:job-type UpdateDependencies
+ :job-key "metabase.task.dependency-backfill.job"
+ :trigger-key "metabase.task.dependency-backfill.trigger"
+ :job-name "Update Dependencies"
+ :batch-size deps.settings/dependency-backfill-batch-size
+ :delay-minutes deps.settings/dependency-backfill-delay-minutes
+ :variance-minutes deps.settings/dependency-backfill-variance-minutes
+ :run-batch! #'run-batch!})
+
+(mu/defn trigger-dependency-job! :- :any
+ "Trigger the UpdateDependencies job, unless a MB_DEPENDENCY_BACKFILL_BATCH_SIZE that isn't positive
+ disables it."
+ []
+ (task.util/trigger-run! job))
+
+(defmethod task/init! ::UpdateDependencies [_]
+ (task.util/init-job! job))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
deleted file mode 100644
index d7b36c65ce3..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
+++ /dev/null
@@ -1,97 +0,0 @@
-(ns metabase-enterprise.dependencies.task.entity-check
- (:require
- [clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.triggers :as triggers]
- [java-time.api :as t]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
- [metabase-enterprise.dependencies.settings :as deps.settings]
- [metabase-enterprise.dependencies.task-util :as deps.task-util]
- [metabase.premium-features.core :as premium-features]
- [metabase.task.core :as task]
- [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(defn- process-one-batch!
- "Process one batch of entities. Returns true if the full batch was used."
- []
- (-> (reduce (fn [batch-size entity-type]
- (if (< batch-size 1)
- (reduced 0)
- (let [processed (deps.findings/analyze-batch! entity-type batch-size)]
- (when (pos? processed)
- (log/info "Updated" processed "entities of type" entity-type))
- (- batch-size processed))))
- (deps.settings/dependency-entity-check-batch-size)
- deps.findings/analyzable-entities)
- (< 1)))
-
-(defn- check-entities!
- "Analyze stale entities in batches until none are left or a pass stops reducing their count."
- []
- (when (premium-features/has-feature? :dependencies)
- (loop [prev-stale nil]
- (process-one-batch!)
- (let [stale (deps.analysis-finding/stale-entity-count)]
- (when (and (pos? stale)
- (or (nil? prev-stale) (< stale prev-stale)))
- (recur stale))))))
-
-(declare schedule-run!)
-
-(defn- reschedule-after-run!
- "Schedule the next run of the entity check job unless the `:dependencies` feature is definitively absent."
- [scheduler]
- (if (false? (premium-features/canonically-has-feature? :dependencies))
- (log/debug "Not rescheduling job Dependency Entity Check: the dependencies feature is not enabled")
- (schedule-run! scheduler
- (deps.task-util/job-delay
- (deps.settings/dependency-entity-check-delay-minutes)
- (deps.settings/dependency-entity-check-variance-minutes)))))
-
-(task/defjob DependencyEntityCheck
- "Check all entities for validity"
- {:saved-class "metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"
- :concurrent? false}
- [ctx]
- (log/info "Executing DependencyEntityCheck job...")
- (check-entities!)
- (reschedule-after-run! (.getScheduler ctx)))
-
-(def ^:private job-key "metabase.dependencies.task.entity-check.job")
-(def ^:private trigger-key "metabase.dependencies.task.entity-check.trigger")
-
-(defn- schedule-run!
- "Schedule a run of the entity check job `delay-in-seconds` from now."
- [scheduler delay-in-seconds]
- (let [start-at (-> (t/instant)
- (t/+ (t/duration delay-in-seconds :seconds))
- java.util.Date/from)
- trigger (triggers/build
- (triggers/with-identity (triggers/key trigger-key))
- (triggers/for-job job-key)
- (triggers/start-at start-at))
- job (jobs/build (jobs/of-type DependencyEntityCheck) (jobs/with-identity job-key))]
- (log/info "Scheduling next run of job Dependency Entity Check at" start-at)
- (task/schedule-task! scheduler job trigger)))
-
-(defmethod task/init! ::DependencyEntityCheck [_]
- (when-not (pos? (deps.settings/dependency-entity-check-batch-size))
- (log/info "Dependency entity check batch size is not positive; the job will run but process nothing"))
- (schedule-run!
- (task/scheduler)
- (deps.task-util/job-initial-delay
- (deps.settings/dependency-entity-check-variance-minutes))))
-
-(defn trigger-entity-check-job!
- "Trigger the DependencyEntityCheck job to run after a brief delay, unless the batch size disables it.
-
- The 1-second delay ensures the calling transaction has committed before the job checks for stale entities. Entity
- changes fire this, so leaving it ungated is what made a disabled job wake roughly once a second; the job's own
- periodic schedule is left alone so it still resumes if the batch size becomes positive.
-
- Disable with MB_DEPENDENCY_ENTITY_CHECK_BATCH_SIZE=0"
- []
- (when (pos? (deps.settings/dependency-entity-check-batch-size))
- (schedule-run! (task/scheduler) 1)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.clj
new file mode 100644
index 00000000000..020ed205c41
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.clj
@@ -0,0 +1,53 @@
+(ns metabase-enterprise.dependencies.task.result-metadata
+ "Implements a task that re-derives the result metadata of the MBQL cards depending on changed entities, see
+ [[metabase-enterprise.dependencies.result-metadata]]."
+ (:require
+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase-enterprise.dependencies.task.util :as task.util]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms])
+ (:import (org.quartz JobExecutionContext)))
+
+(set! *warn-on-reflection* true)
+
+(declare ^:private job)
+
+(mu/defn- run-batch! :- [:maybe ms/PositiveInt]
+ "Re-derive a batch of stale cards, returning when to run again: a second from now when it took any, or nil."
+ []
+ (when (premium-features/has-feature? :dependencies)
+ (let [taken (deps.result-metadata/update-result-metadata! (deps.settings/dependency-result-metadata-batch-size))]
+ (when (pos? taken)
+ (log/debug "Processed" taken "entities for result metadata")
+ 1))))
+
+(task/defjob UpdateResultMetadata
+ "Re-derive the result metadata of the MBQL cards depending on changed entities."
+ {:saved-class "metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata"
+ :concurrent? false}
+ [ctx]
+ (log/debug "Executing UpdateResultMetadata job...")
+ (task.util/run-and-reschedule! (.getScheduler ^JobExecutionContext ctx) job))
+
+(def ^:private job
+ {:job-type UpdateResultMetadata
+ :job-key "metabase.dependencies.task.result-metadata.job"
+ :trigger-key "metabase.dependencies.task.result-metadata.trigger"
+ :job-name "Update Result Metadata"
+ :batch-size deps.settings/dependency-result-metadata-batch-size
+ :delay-minutes deps.settings/dependency-result-metadata-delay-minutes
+ :variance-minutes deps.settings/dependency-result-metadata-variance-minutes
+ :run-batch! #'run-batch!})
+
+(mu/defn trigger-result-metadata-job! :- :any
+ "Trigger the UpdateResultMetadata job, unless a MB_DEPENDENCY_RESULT_METADATA_BATCH_SIZE that isn't positive
+ disables it."
+ []
+ (task.util/trigger-run! job))
+
+(defmethod task/init! ::UpdateResultMetadata [_]
+ (task.util/init-job! job))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/util.clj
new file mode 100644
index 00000000000..8d9b5ffc097
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/util.clj
@@ -0,0 +1,115 @@
+(ns metabase-enterprise.dependencies.task.util
+ "The scheduling the dependencies module's self-rescheduling jobs share, each job described by a map of its
+ `:job-type`, `:job-key`, `:trigger-key` and `:job-name`, the getters of its `:batch-size`, `:delay-minutes` and
+ `:variance-minutes` settings, and `:run-batch!`, which processes a batch and returns how many seconds from now to
+ run again, or nil to stop."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms])
+ (:import
+ (java.util Date)
+ (org.quartz Scheduler)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private Job
+ "A self-rescheduling job, as the namespace docstring describes it."
+ [:map {:closed true}
+ [:job-type [:fn {:error/message "a job class"} class?]]
+ [:job-key :string]
+ [:trigger-key :string]
+ [:job-name :string]
+ [:batch-size [:fn {:error/message "a setting getter"} ifn?]]
+ [:delay-minutes [:fn {:error/message "a setting getter"} ifn?]]
+ [:variance-minutes [:fn {:error/message "a setting getter"} ifn?]]
+ [:run-batch! [:fn {:error/message "a function"} ifn?]]])
+
+(def ^:private MaybeScheduler
+ "A Quartz scheduler, or nil when none is running."
+ [:maybe (ms/InstanceOfClass Scheduler)])
+
+(mu/defn- job-delay :- ms/IntGreaterThanOrEqualToZero
+ "A random delay in seconds between `delay - variance` and `delay + variance` minutes."
+ [delay :- :int
+ variance :- :int]
+ (let [delay-duration (t/duration delay :minutes)
+ variance-duration (t/duration variance :minutes)]
+ (max 0 (.. delay-duration
+ (minus variance-duration)
+ (plusSeconds (rand-int (.. variance-duration
+ (multipliedBy 2)
+ toSeconds)))
+ toSeconds))))
+
+(mu/defn next-run-delay :- ms/IntGreaterThanOrEqualToZero
+ "A random delay in seconds before `job` runs again after a failure, within its variance of its delay."
+ [{:keys [delay-minutes variance-minutes]} :- Job]
+ (job-delay (delay-minutes) (variance-minutes)))
+
+(mu/defn- next-fire-time :- [:maybe (ms/InstanceOfClass Date)]
+ "When the trigger with `trigger-key` fires next on `scheduler`, or nil when it is not pending."
+ ^Date [^Scheduler scheduler :- MaybeScheduler
+ trigger-key :- :string]
+ (some-> scheduler (.getTrigger (triggers/key trigger-key)) .getNextFireTime))
+
+(def ^:private schedule-lock
+ "The lock that makes reading and replacing a job's pending trigger one step for this instance."
+ (Object.))
+
+(mu/defn schedule-run! :- :any
+ "Schedule a run of `job` `delay-in-seconds` from now on `scheduler`, leaving a run already pending before then in
+ place, or a second from now when that pending run is past due, since Quartz can leave a trigger past due for
+ minutes."
+ [scheduler :- MaybeScheduler
+ {:keys [job-type job-key trigger-key job-name]} :- Job
+ delay-in-seconds :- ms/IntGreaterThanOrEqualToZero]
+ (locking schedule-lock
+ (let [now (t/instant)
+ start-at (t/+ now (t/duration delay-in-seconds :seconds))
+ pending (some-> (next-fire-time scheduler trigger-key) .toInstant)]
+ (when-not (and pending (t/after? pending now) (not (t/after? pending start-at)))
+ (let [start-at (Date/from (if (or (nil? pending) (t/after? pending start-at))
+ start-at
+ (t/+ now (t/duration 1 :seconds))))
+ trigger (triggers/build
+ (triggers/with-identity (triggers/key trigger-key))
+ (triggers/for-job job-key)
+ (triggers/start-at start-at))
+ job (jobs/build (jobs/of-type job-type) (jobs/with-identity job-key))]
+ (log/debug "Scheduling next run of job" job-name "at" start-at)
+ (task/schedule-task! scheduler job trigger))))))
+
+(mu/defn run-and-reschedule! :- :any
+ "Run a batch of `job` unless a batch size that isn't positive disables it, then schedule its next run when the batch
+ asks for one, and after [[next-run-delay]] when it throws, before rethrowing, so a failure doesn't stop it until the
+ next write."
+ [scheduler :- MaybeScheduler
+ {:keys [run-batch! job-name batch-size] :as job} :- Job]
+ (if (pos? (batch-size))
+ (try
+ (if-some [delay-in-seconds (run-batch!)]
+ (schedule-run! scheduler job delay-in-seconds)
+ (log/debug "Job" job-name "has nothing more to do, stopping."))
+ (catch Throwable e
+ (schedule-run! scheduler job (next-run-delay job))
+ (throw e)))
+ (log/debug "Job" job-name "has a batch size that isn't positive, stopping.")))
+
+(mu/defn trigger-run! :- :any
+ "Schedule a run of `job` a second from now, unless a batch size that isn't positive disables it."
+ [{:keys [batch-size] :as job} :- Job]
+ (when (pos? (batch-size))
+ (schedule-run! (task/scheduler) job 1)))
+
+(mu/defn init-job! :- :any
+ "Schedule the first run of `job` after startup, at a random time within its variance, unless a batch size that isn't
+ positive disables it."
+ [{:keys [batch-size variance-minutes job-name] :as job} :- Job]
+ (if (pos? (batch-size))
+ (schedule-run! (task/scheduler) job (rand-int (.toSeconds (t/duration (variance-minutes) :minutes))))
+ (log/info "The batch size of job" job-name "is not positive; the job is disabled")))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj
deleted file mode 100644
index 8efa5ffb6e5..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj
+++ /dev/null
@@ -1,27 +0,0 @@
-(ns metabase-enterprise.dependencies.task-util
- (:require
- [java-time.api :as t]))
-
-(set! *warn-on-reflection* true)
-
-(defn job-delay
- "Generates a random delay in seconds based on `delay` and `variance`.
-
- Both `delay` and `variance` should be in minutes, and the result will vary between `delay-variance` and
- `delay+variance`."
- [delay variance]
- (let [delay-duration (t/duration delay :minutes)
- variance-duration (t/duration variance :minutes)]
- (max 0 (.. delay-duration
- (minus variance-duration)
- (plusSeconds (rand-int (.. variance-duration
- (multipliedBy 2)
- toSeconds)))
- toSeconds))))
-
-(defn job-initial-delay
- "Generates a random delay in seconds based on `variance`.
-
- `variance` is in minutes and the result will vary between 0 and `variance`"
- [variance]
- (rand-int (.toSeconds (t/duration variance :minutes))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index fcbaa74296f..30c26ef859b 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -9,7 +9,6 @@
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.app-db.core :as mdb]
[metabase.app-db.transient-error :as transient-error]
- [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
@@ -288,20 +287,17 @@
:files file-names}
(first ingest-errors)))))
(log/infof "Starting deserialization, total %s documents" (count contents))
- (try
- (reduce (fn [ctx item]
- (try
- (load-one! ctx item)
- (catch Exception e
- (when-not continue-on-error
- (throw e))
- ;; eschew big and scary stacktrace
- (log/warnf (u/strip-error e "Skipping deserialization error"))
- (update ctx :errors conj e))))
- ctx
- contents)
- (finally
- (dependencies.hooks/after-serdes-load))))
+ (reduce (fn [ctx item]
+ (try
+ (load-one! ctx item)
+ (catch Exception e
+ (when-not continue-on-error
+ (throw e))
+ ;; eschew big and scary stacktrace
+ (log/warnf (u/strip-error e "Skipping deserialization error"))
+ (update ctx :errors conj e))))
+ ctx
+ contents))
(when reindex?
;; Reindex after all entities are loaded. Individual entity commits may have produced stale
;; search index entries; this ensures the index reflects the final state.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 92a174293f1..ca6dd888b74 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -76,6 +76,7 @@
"AiUsageLog"
"AnalysisFinding"
"AnalysisFindingError"
+ "AnalysisFindingStatus"
"ApiKey"
"ApiKeyUsageLog"
"ApplicationPermissionsRevision"
@@ -158,6 +159,7 @@
"ReplacementRun"
"RemoteSyncObject"
"RemoteSyncTask"
+ "ResultMetadataStatus"
"Revision"
"Sandbox"
"SearchIndexMetadata"
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
index ef97f08d808..94729b701b9 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
@@ -3,7 +3,6 @@
[clojure.test :refer :all]
[metabase-enterprise.data-apps.group-access :as group-access]
[metabase-enterprise.dependencies.test-util :as deps.test]
- [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.permissions-rest.data-permissions.graph :as data-perms.graph]
@@ -50,7 +49,7 @@
:model/QueryAction _ {:action_id action-id
:database_id (mt/id)
:dataset_query (lib/native-query mp "DELETE FROM CHECKINS WHERE ID = 1")}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
(group-access/add-groups! app [(:id group)])
(is (=? [{:group_id (:id group)
@@ -72,48 +71,29 @@
:model/PermissionsGroup group {}]
(perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
(group-access/add-groups! app [(:id group)])
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]
(group-access/permission-warnings app)))))))))))
(deftest group-permission-warnings-follow-snippet-edit-test
(testing "after a snippet edit and the jobs, the warnings list the tables of the snippet's new SQL"
- (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
- (mt/with-no-data-perms-for-all-users!
- (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]
- (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}
- :model/DataApp {collection-id :resource_collection_id :as app}
- {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
- :model/Card _ {:collection_id collection-id
- :dataset_query (lib/native-query (mt/metadata-provider)
- "SELECT * FROM {{snippet: birds source}}")}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
- (deps.test/synchronously-run-backfill!)
- (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})
- (deps.test/synchronously-run-backfill!)
- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}]
- (group-access/permission-warnings app)))))))))
-
-(deftest group-permission-warnings-include-table-synced-after-analysis-test
- (testing "a table that sync finds after the app's card was analyzed enters the warnings once the jobs run"
- (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
- (mt/with-no-data-perms-for-all-users!
- (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]
- (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}
- {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
- :model/Card _ {:collection_id collection-id
- :dataset_query (lib/native-query (mt/metadata-provider)
- "SELECT * FROM PUBLIC.BIRDS_LATE")}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
- (deps.test/synchronously-run-backfill!)
- (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]
- (dependencies.hooks/after-database-sync (mt/id))
- (deps.test/synchronously-run-backfill!)
- (is (=? [{:group_id (:id group) :missing_tables [{:id late-id}]}]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]
+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}
+ :model/DataApp {collection-id :resource_collection_id :as app}
+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card _ {:collection_id collection-id
+ :dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM {{snippet: birds source}}")}
+ :model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (deps.test/synchronously-run-dependency-job!)
+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}]
(group-access/permission-warnings app))))))))))
(deftest group-permission-warnings-exclude-inactive-tables-test
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.clj
new file mode 100644
index 00000000000..873aef424e2
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.clj
@@ -0,0 +1,584 @@
+(ns metabase-enterprise.dependencies.analysis-finding-test
+ (:require
+ [clojure.test :refer :all]
+ [medley.core :as m]
+ [metabase-enterprise.dependencies.analysis :as deps.analysis]
+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.hooks :as deps.hooks]
+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.models.analysis-finding-error :as models.analysis-finding-error]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
+ [metabase.query-processor :as qp]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(defn- run-jobs!
+ "Run the dependency job, whose current dependencies the analysis job waits for, then the analysis job, until each has
+ nothing left to do."
+ []
+ (deps.test/synchronously-run-dependency-job!)
+ (mt/with-premium-features #{:dependencies}
+ (deps.test/run-analysis-finding-job!)))
+
+(defn- backfill-all-entity-analyses! []
+ (run-jobs!))
+
+(defn- status [entity-type entity-id]
+ (t2/select-one :model/AnalysisFindingStatus :entity_type entity-type :entity_id entity-id))
+
+(defn- analyzed-ids!
+ "The IDs of the cards `thunk` analyzes, in order."
+ [thunk]
+ (let [analyzed (atom [])
+ upsert! (mt/original-fn #'models.analysis-finding/upsert-analysis!)]
+ (mt/with-dynamic-fn-redefs [models.analysis-finding/upsert-analysis!
+ (fn [entity-type entity-id & args]
+ (when (= :card entity-type)
+ (swap! analyzed conj entity-id))
+ (apply upsert! entity-type entity-id args))]
+ (thunk))
+ @analyzed))
+
+(deftest ^:synchronized can-analyze-entity-batches-test
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products-id (mt/id :products)
+ orders-id (mt/id :orders)
+ products (lib.metadata/table mp products-id)
+ orders (lib.metadata/table mp orders-id)]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]
+ (run-jobs!)
+ (is (= #{models.analysis-finding/*current-analysis-finding-version*}
+ (t2/select-fn-set :analysis_version
+ :model/AnalysisFindingStatus
+ :entity_id [:in [card-id other-card-id]]
+ :entity_type :card)))))))
+
+(deftest ^:synchronized does-not-repeatedly-analyze-entities-test
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products-id (mt/id :products)
+ orders-id (mt/id :orders)
+ products (lib.metadata/table mp products-id)
+ orders (lib.metadata/table mp orders-id)]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]
+ (is (= #{card-id other-card-id} (set (filter #{card-id other-card-id} (analyzed-ids! run-jobs!)))))
+ (is (empty? (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))))))
+
+(deftest ^:synchronized re-analyze-entities-when-analysis-version-bumped-test
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]
+ (run-jobs!)
+ (is (empty? (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))
+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
+ (is (= #{card-id other-card-id} (set (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))))))))
+
+(deftest ^:synchronized does-analyze-native-entities-test
+ (testing "failed analysis result is appropriately stored in the appdb"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]
+ (run-jobs!)
+ (is (= models.analysis-finding/*current-analysis-finding-version*
+ (:analysis_version (status :card card-id))))
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_id card-id
+ :entity_type :card))))))))
+
+(deftest ^:synchronized does-report-errors-for-missing-refs-in-fields-test
+ (testing "missing (not inactive) field refs in :fields are reported as findings (GHY-3157)"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ base (lib/query mp orders)
+ cols (vec (take 5 (lib/returned-columns base orders)))
+ bad-col (-> (first cols)
+ lib/ref
+ (assoc 2 "bad_column"))
+ query (lib/with-fields base (conj cols bad-col))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
+ (run-jobs!)
+ (is (= models.analysis-finding/*current-analysis-finding-version*
+ (:analysis_version (status :card card-id))))
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_id card-id
+ :entity_type :card))))))))
+
+;; TODO: (bshepherdson, 2026-02-05) Add a test like does-not-report-errors-in-removable-refs-test-1-stage-fields for
+;; join clause :fields as well. See QUE-3081 and QUE-3044.
+
+(deftest ^:parallel reports-missing-field-in-downstream-card-fields-test
+ (testing "Q2 based on Q1 with :fields referencing a column Q1 no longer returns should be reported (GHY-3157)"
+ (let [mp meta/metadata-provider
+ q1 (lib/query mp (meta/table-metadata :orders))
+ q1-cols (lib/returned-columns q1)
+ removed-col (m/find-first #(= (:name %) "ID") q1-cols)
+ ;; Card 101 = Q1 with ID removed from result-metadata (simulates Q1 dropping that field)
+ mp (lib.tu/metadata-provider-with-card-from-query
+ mp 101 q1
+ {:result-metadata (vec (remove #(= (:name %) "ID") q1-cols))})
+ ;; Q2: query based on card 101, with :fields including the now-missing ID
+ q2-base (lib/query mp {:lib/type :metadata/card :id 101})
+ q2-cols (lib/returned-columns q2-base)
+ q2 (lib/with-fields q2-base (conj (vec q2-cols) (lib/ref removed-col)))
+ ;; Card 102 = Q2 (provide result-metadata explicitly since q2 has a bad ref)
+ mp (lib.tu/metadata-provider-with-card-from-query
+ mp 102 q2
+ {:result-metadata (vec q2-cols)})
+ results (deps.analysis/check-entity mp :card 102)]
+ (is (seq results)
+ "Q2 should have findings for the missing ID column from Q1"))))
+
+(deftest ^:synchronized analyze-batch-picks-up-missing-analyses-test
+ (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
+ (testing "card has no analysis finding initially"
+ (is (not (t2/exists? :model/AnalysisFinding
+ :entity_type :card
+ :entity_id card-id))))
+ (testing "the jobs create the analysis of the card"
+ (run-jobs!))
+ (testing "card now has an analysis finding"
+ (is (t2/exists? :model/AnalysisFinding
+ :entity_type :card
+ :entity_id card-id))))))))
+
+(deftest ^:synchronized failed-analysis-waits-for-change-test
+ (testing "an entity whose analysis keeps throwing records a failed finding, and is given up on until it changes"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFindingError]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (fn [_] (throw (ex-info "Broken" {})))}
+ run-jobs!)
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_type :card :entity_id card-id)))
+ (is (empty? (filter #{card-id} (analyzed-ids! run-jobs!))))))))))
+
+(deftest ^:synchronized analysis-re-analyzes-changed-entity-and-direct-dependents-test
+ (testing "after a card's query changes, the analysis finding job re-analyzes it and the cards directly depending on
+ it, and nothing for a question's rename"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {gp-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {p-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}
+ :model/Card {c-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id p-id
+ :to_entity_type :card :to_entity_id gp-id}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id c-id
+ :to_entity_type :card :to_entity_id p-id}]
+ (run-jobs!)
+ (t2/update! :model/Card gp-id {:name "Renamed"})
+ (is (empty? (filter #{gp-id p-id c-id} (analyzed-ids! run-jobs!))))
+ (t2/update! :model/Card gp-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})
+ (is (= #{gp-id p-id}
+ (set (filter #{gp-id p-id c-id} (analyzed-ids! run-jobs!)))))))))))
+
+(deftest ^:synchronized analysis-follows-table-field-write-test
+ (testing "a write of a field of a table re-analyzes the entities depending on the table"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :products)))}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :products)}]
+ (run-jobs!)
+ (is (not (deps.test/analysis-finding-pending? :card card-id)))
+ (mt/with-temp-vals-in-db :model/Field (mt/id :products :title) {:description "Changed"}
+ (is (deps.test/analysis-finding-pending? :table (mt/id :products)))
+ (is (some #{card-id} (analyzed-ids! run-jobs!))))))))))
+
+(deftest ^:synchronized cyclic-dependency-drain-terminates-test
+ (testing "the analysis finding job terminates on a cyclic dependency graph"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {a-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {b-id :id} {:dataset_query (lib/query mp products)}
+ ;; cycle: a depends on b, and b depends on a
+ :model/Dependency _ {:from_entity_type :card :from_entity_id a-id
+ :to_entity_type :card :to_entity_id b-id}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id b-id
+ :to_entity_type :card :to_entity_id a-id}]
+ (let [done (future (run-jobs!))]
+ (is (not= ::timeout (deref done 30000 ::timeout))
+ "the analysis finding job must terminate on a cyclic graph")
+ (is (not (deps.test/run-analysis-finding-job!)) "and leave nothing to do")))))))
+
+(deftest ^:synchronized python-transform-records-clean-finding-test
+ (testing "GHY-3584: a Python transform has no query, so upsert-analysis! resolves its database from source_database_id and records a clean finding"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFindingError]
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :python
+ :source-database (mt/id)
+ :source-tables []
+ :body "def transform():\n pass"}}]
+ (lib-be/with-metadata-provider-cache
+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Transform transform-id)))
+ (is (=? {:result true}
+ (t2/select-one :model/AnalysisFinding :entity_type :transform :entity_id transform-id)))
+ (is (empty? (models.analysis-finding-error/errors-for-entity :transform transform-id))))))))
+
+(deftest ^:synchronized unanalyzable-entity-records-error-test
+ (testing "an entity whose database can't be resolved records an error finding instead of no-oping forever"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFindingError]
+ (mt/with-temp [:model/Card card {:dataset_query (mt/native-query {:query "SELECT 1"})
+ :database_id (mt/id)}]
+ (let [cid (:id card)]
+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (constantly nil)}
+ (fn []
+ (lib-be/with-metadata-provider-cache
+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))))
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_type :card :entity_id cid)))
+ (is (seq (models.analysis-finding-error/errors-for-entity :card cid))
+ "an error finding is recorded explaining why the entity couldn't be analyzed")))))))
+
+(deftest ^:synchronized unanalyzable-entity-rechecks-when-database-resolvable-again-test
+ (testing "after a no-database error, the entity is analyzed normally once its database resolves again"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFindingError]
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp products)}]
+ (let [cid (:id card)]
+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (constantly nil)}
+ (fn []
+ (lib-be/with-metadata-provider-cache
+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))))
+ (is (seq (models.analysis-finding-error/errors-for-entity :card cid)))
+ (lib-be/with-metadata-provider-cache
+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))
+ (is (empty? (models.analysis-finding-error/errors-for-entity :card cid))
+ "re-analyzed normally: the database error is replaced by a clean result"))))))))
+
+(deftest ^:synchronized analyze-batch-shares-one-metadata-provider-test
+ (testing "Every entity in an analysis batch must share one MetadataProvider per database. Reading an entity attaches
+ a provider to its query, so selecting the batch outside the cache gives each entity a private one. Nothing
+ reads through those providers today -- upsert-analysis! builds its own from the database id and passes it
+ explicitly -- so this is a latent trap rather than a live leak: the moment an entity is analyzed through its
+ own query, a batch of 500 retains 500 providers' worth of fetched metadata. upsert-analysis! already throws
+ when no cache is bound, so the cache is meant to cover this work; only the select sat outside it."
+ (backfill-all-entity-analyses!)
+ (let [providers (atom [])
+ native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}
+ :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]
+ (let [wanted #{card1-id card2-id}
+ upsert-analysis! (mt/original-fn #'deps.analysis-finding/upsert-analysis!)]
+ (mt/with-dynamic-fn-redefs [deps.analysis-finding/upsert-analysis!
+ (fn [entity & args]
+ (when (wanted (:id entity))
+ (swap! providers conj (:lib/metadata (:dataset_query entity))))
+ (apply upsert-analysis! entity args))]
+ (run-jobs!)))
+ (testing "both cards were selected, so the assertion below is meaningful"
+ (is (= 2 (count @providers))))
+ (testing "and they carry the same provider instance"
+ ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be
+ ;; satisfied by two separate caches and would not detect the regression.
+ (is (every? #(identical? (first @providers) %) @providers))))))))
+
+(defn- analyzed-by-job!
+ "The entities, as `[entity-type entity-id]`, that running the analysis finding job now analyzes, after the dependency
+ job it waits for."
+ []
+ (deps.test/synchronously-run-dependency-job!)
+ (let [analyzed (atom #{})
+ upsert! (mt/original-fn #'models.analysis-finding/upsert-analysis!)]
+ (mt/with-dynamic-fn-redefs [models.analysis-finding/upsert-analysis!
+ (fn [entity-type entity-id & args]
+ (swap! analyzed conj [entity-type entity-id])
+ (apply upsert! entity-type entity-id args))]
+ (deps.test/run-analysis-finding-job!))
+ @analyzed))
+
+(defn- analyzed-among!
+ "Which of the entities `nodes`, `[entity-type entity-id]`, running the analysis finding job now analyzes."
+ [& nodes]
+ (into #{} (filter (set nodes)) (analyzed-by-job!)))
+
+(defn- assert-has-analyses
+ "Asserts that a given set of entities have the appropriate analysis version.
+
+ Format is `{entity-type {entity-id analysis-version}}`.
+
+ Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version.
+
+ Use nil for analysis-version if you want to check that no analysis exists."
+ [spec]
+ (doseq [[entity-type ids-and-versions] spec]
+ (let [analyses-map (->> (t2/select [:model/AnalysisFindingStatus :entity_id :analysis_version]
+ :entity_type entity-type)
+ (into {} (map (juxt :entity_id :analysis_version))))]
+ (doseq [[id expected-version] ids-and-versions]
+ (testing (str "Checking " entity-type " " id)
+ (if (= expected-version -1)
+ (is (contains? analyses-map id))
+ (is (= expected-version (analyses-map id)))))))))
+
+(deftest ^:synchronized card-update-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Card updates should mark the card stale and trigger the background job"
+ (let [products-id (mt/id :products)
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-card-id
+ :to_entity_type :card
+ :to_entity_id parent-card-id}]
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})
+ (assert-has-analyses
+ {:card {parent-card-id nil}})
+ (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFindingStatus
+ :entity_type :card
+ :entity_id child-card-id))
+ "an update writes no analysis record")))))))
+
+(deftest ^:synchronized card-update-updates-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "changing a card's query has the job re-analyze it and its dependents"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-card-id
+ :to_entity_type :card
+ :to_entity_id parent-card-id}]
+ (analyzed-by-job!)
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (= #{[:card parent-card-id] [:card child-card-id]}
+ (analyzed-among! [:card parent-card-id] [:card child-card-id])))))))))
+
+(deftest ^:synchronized card-update-triggers-native-cards-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a native card's update with new run metadata has the job re-analyze it and its direct dependents"
+ (mt/with-temp [:model/Card {parent-id :id} {:dataset_query (lib/native-query mp "select * from products")}
+ :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-id
+ :to_entity_type :card :to_entity_id parent-id}]
+ (analyzed-by-job!)
+ (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")
+ :result_metadata (get-in (qp/process-query (lib/native-query mp "select * from orders"))
+ [:data :results_metadata :columns])})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (= #{[:card parent-id] [:card child-id]}
+ (analyzed-among! [:card parent-id] [:card child-id]))))))))
+
+(deftest ^:synchronized card-update-analyzes-dependents-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a card update has the job re-analyze the card and its dependents, transforms included"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}]
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/query mp (lib.metadata/card
+ (mt/metadata-provider)
+ parent-card-id))}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]
+ (analyzed-by-job!)
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
+ (is (= #{[:card parent-card-id] [:transform transform-id]}
+ (analyzed-among! [:card parent-card-id] [:transform transform-id] [:card other-card-id]))))))))))
+
+(deftest ^:synchronized transform-update-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Transform update does not error when no analyses exist yet"
+ (let [products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (is (= 1 (t2/update! :model/Transform transform-id
+ {:source {:type :query
+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))
+
+(deftest ^:synchronized transform-update-updates-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a transform update has the job re-analyze it"
+ (let [products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (analyzed-by-job!)
+ (t2/update! :model/Transform transform-id {:source {:type :query
+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})
+ (is (= #{[:transform transform-id]} (analyzed-among! [:transform transform-id])))))))))
+
+(deftest ^:synchronized transform-update-triggers-native-transforms-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Native transform updates do not error"
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/native-query mp "select * from products")}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (is (= 1 (t2/update! :model/Transform transform-id
+ {:source {:type :query
+ :query (lib/native-query mp "select * from orders")}}))))))))
+
+(deftest ^:synchronized transform-run-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Transform run completes without error when no analyses exist"
+ (let [products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public" :name "sample" :type :table}}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :transform :to_entity_id transform-id}]
+ (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))
+
+(deftest ^:synchronized transform-run-without-table-analyzes-nothing-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a transform run re-analyzes nothing by itself, since the sync of its output table marks what reads that table"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public" :name "sample" :type :table}}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :transform :to_entity_id transform-id}]
+ (analyzed-by-job!)
+ (deps.hooks/after-transform-run transform-id nil)
+ (is (= #{}
+ (analyzed-among! [:card card-id] [:transform transform-id] [:card other-card-id])))))))))
+
+(deftest ^:synchronized segment-update-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [_]
+ (testing "Segment update does not error when no analyses exist yet"
+ (let [products-id (mt/id :products)
+ price-field-id (mt/id :products :price)]
+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ (is (= 1 (t2/update! :model/Segment segment-id
+ {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))
+
+(deftest ^:synchronized segment-update-analyzes-segment-and-dependents-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a segment update has the job re-analyze the segment and the cards using it"
+ (let [products-id (mt/id :products)
+ price-field-id (mt/id :products :price)
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/filter (lib/query mp products)
+ (lib.metadata/segment (mt/metadata-provider)
+ segment-id))}]
+ (analyzed-by-job!)
+ (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})
+ (is (= #{[:segment segment-id] [:card card-id]}
+ (analyzed-among! [:segment segment-id] [:card card-id]))))))))))
+
+(deftest ^:synchronized table-metadata-update-triggers-dependent-analysis-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a table metadata update has the job re-analyze the table's dependents, not unrelated cards"
+ (let [products-id (mt/id :products)
+ orders (lib.metadata/table mp (mt/id :orders))
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
+ (analyzed-by-job!)
+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
+ (is (= #{[:card card-id]} (analyzed-among! [:card card-id] [:card other-card-id]))))))))))
+
+(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Table metadata update does not error when no findings exist yet"
+ (let [products-id (mt/id :products)
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
+ (assert-has-analyses {:card {card-id nil}}))))))))
+
+(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "a field metadata update has the job re-analyze the dependents of the field's table"
+ (let [products-id (mt/id :products)
+ orders (lib.metadata/table mp (mt/id :orders))
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
+ (analyzed-by-job!)
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category)
+ :display_name "Category renamed"}]
+ (is (= #{[:card card-id]} (analyzed-among! [:card card-id] [:card other-card-id]))))))))))
+
+(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test
+ (deps.test/run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Field metadata update does not error when no findings exist yet"
+ (let [products-id (mt/id :products)
+ products (lib.metadata/table mp products-id)]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]
+ (assert-has-analyses {:card {card-id nil}}))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 8bd8650d9e0..c3e6b382dd6 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -2,10 +2,9 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.api :as deps.api]
- [metabase-enterprise.dependencies.findings :as dependencies.findings]
[metabase-enterprise.dependencies.hooks :as deps.hooks]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.collections.models.collection :as collection]
[metabase.collections.test-utils :refer [personal-collection-id]]
@@ -128,7 +127,7 @@
"Run analysis for a specific card to detect broken references.
Must be called within lib-be/with-metadata-provider-cache."
[card-id]
- (dependencies.findings/upsert-analysis! (t2/select-one :model/Card :id card-id)))
+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Card :id card-id)))
(deftest graph-test
(testing "GET /api/ee/dependencies/graph"
@@ -137,7 +136,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user)
{card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph" :id card-id-2 :type "card")
creator {:email "[email protected]"
:id (:id user)}]
@@ -198,7 +197,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [_card-1 (card/create-card! (basic-card "Card 1" :orders) user)
_card-2 (card/create-card! (basic-card "Card 2" :orders) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
:id (mt/id :orders)
:type "table")]
@@ -247,7 +246,7 @@
:target {:type "table" :name (mt/random-name)}}]
(events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id products-id :type "table")
table-node (first (filter #(= (:id %) products-id) (:nodes response)))]
@@ -267,7 +266,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user)
{card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id card-id-1
:type "card"
@@ -307,7 +306,7 @@
:size_x 4
:size_y 4})
(events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "single dependent-types value (backward compatibility)"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id card-id-1
@@ -339,7 +338,7 @@
(let [{dependency-card-id :id :as dependency-card} (card/create-card! (basic-card "Base card - cardtypes") user)
_question-card (card/create-card! (assoc (wrap-card dependency-card) :name "Question card") user)
_model-card (card/create-card! (assoc (wrap-card dependency-card) :name "Model card" :type :model) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "single dependent-card-types value"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id dependency-card-id
@@ -381,7 +380,7 @@
dependent-card (card/create-card! (wrap-card base-card) user)]
(card/update-card! {:card-before-update base-card
:card-updates {:archived true}})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
:id (:id dependent-card)
:type "card")
@@ -403,7 +402,7 @@
dependent-card (card/create-card! (wrap-card base-card) user)]
(card/update-card! {:card-before-update base-card
:card-updates {:archived true}})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
:id (:id dependent-card)
:type "card")
@@ -426,7 +425,7 @@
dependent-card (card/create-card! (wrap-card base-card) user)]
(card/update-card! {:card-before-update dependent-card
:card-updates {:archived true}})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id (:id base-card)
:type "card"
@@ -450,7 +449,7 @@
;; Simulate the transform having been run: link it to its (now inactive) target table
(t2/update! :model/Transform transform-id {:target_table_id table-id})
(deps.hooks/after-transform-run transform-id table-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "GET /graph/dependents returns the inactive table as a child"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:id transform-id
@@ -479,7 +478,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card)
(let [next-card (create-dependent-card-on-model! user model-card "Another Dependent Card - brokentest" {:table :products})]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id next-card))
(run-analysis-for-card! (:id dependent-card))
(let [response2 (mt/user-http-request :crowberto :get 200 (str "ee/dependencies/graph/dependents?broken=true&type=card&id=" (:id model-card)))]
@@ -503,8 +502,8 @@
:table "ORDERS"
:table_id (mt/id :orders)}]
:body "def transform(orders):\n return orders"}}]
- (deps.test/synchronously-run-backfill!)
- (#'task.entity-check/check-entities!)
+ (deps.test/synchronously-run-dependency-job!)
+ (deps.test/run-analysis-finding-job!)
(let [dependent-ids (fn [broken?]
(set (map :id (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:type "table"
@@ -524,7 +523,7 @@
:model/User user {:email "[email protected]"}]
(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "Returns 403 when user lacks read permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/dependencies/graph"
@@ -544,7 +543,7 @@
:model/User user {:email "[email protected]"}]
(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [{card-id :id} (card/create-card! (assoc (basic-card) :collection_id (:id collection)) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "Returns 403 when user lacks read permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/dependencies/graph/dependents"
@@ -576,7 +575,7 @@
top-card (card/create-card! (assoc (wrap-two-cards readable-base unreadable-base)
:collection_id (:id readable-collection))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees complete upstream graph through readable path"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -617,7 +616,7 @@
unreadable-dependent (card/create-card! (assoc (wrap-card base-card)
:collection_id (:id unreadable-collection))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only readable dependents"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
@@ -649,7 +648,7 @@
end-card (card/create-card! (assoc (wrap-two-cards unreadable-middle readable-alternate)
:collection_id (:id readable-collection))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "Diamond pattern: complete upstream graph via readable path"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -694,7 +693,7 @@
top-card (card/create-card! (assoc (wrap-card unreadable-base)
:collection_id (:id readable-collection))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only the top card in the graph"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -742,7 +741,7 @@
:visualization_settings {}
:collection_id (:id unreadable-collection)}
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only readable cards as dependents of the snippet"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
@@ -786,7 +785,7 @@
(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/create-queries :no)
(let [card (card/create-card! (assoc (basic-card "Card on orders")
:collection_id (:id readable-collection)) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only the card, table is filtered out"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -813,7 +812,7 @@
dashboard-card (card/create-card! (assoc (wrap-card base-card)
:dashboard_id (:id dashboard))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "GET /api/ee/dependencies/graph returns dashboard with :id and :name"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id (:id dashboard-card)
@@ -846,7 +845,7 @@
document-card (card/create-card! (assoc (wrap-card base-card)
:document_id (:id document))
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "GET /api/ee/dependencies/graph returns document with :id and :name"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id (:id document-card)
@@ -879,7 +878,7 @@
:table_id products-id
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id segment-id
:type "segment")
@@ -908,7 +907,7 @@
query-with-segment (-> (lib/query mp products)
(lib/filter (lib.metadata/segment mp segment-id)))
card (card/create-card! (card-with-query "Card using segment" query-with-segment) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:id segment-id
:type "segment"
@@ -960,7 +959,7 @@
:card-id model-id}}))}]
(mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [dependents (fn [user]
(->> (mt/user-http-request user :get 200 "ee/dependencies/graph/dependents"
:id model-id
@@ -988,7 +987,7 @@
:definition (-> (lib/query mp products)
(lib/aggregate (lib/sum price)))}]
(events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-id
:type "measure")
@@ -1014,7 +1013,7 @@
:definition (-> (lib/query mp products)
(lib/aggregate (lib/sum price)))}]
(events/publish-event! :event/measure-create {:object measure-a :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [mp' (mt/metadata-provider)]
(mt/with-temp [:model/Measure {measure-b-id :id :as measure-b} {:name "Measure B"
:table_id products-id
@@ -1022,7 +1021,7 @@
(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)
(lib/sum rating))))}]
(events/publish-event! :event/measure-create {:object measure-b :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-b-id
:type "measure")
@@ -1058,7 +1057,7 @@
query-with-measure (-> (lib/query mp' products)
(lib/aggregate (lib.metadata/measure mp' measure-id)))
card (card/create-card! (card-with-query "Card using measure" query-with-measure) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:id measure-id
:type "measure"
@@ -1116,7 +1115,7 @@
;; Archive measure B (the middle of the chain)
(mt/user-http-request :crowberto :put 200 (str "measure/" measure-b-id)
{:archived true :revision_message "Archive middle measure"})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "after archiving measure B, it and measure A are excluded (chain broken)"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-c-id
@@ -1188,7 +1187,7 @@
(lib/query mp))}]
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id referenced-card-id) :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id unreffed-card-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=unreftest")]
(is (=? {:data [{:id unreffed-card-id
:type "card"
@@ -1205,7 +1204,7 @@
:type :question
:dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}]
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=unreftest")]
(is (=? {:data [{:id unreffed-table-id
:type "table"
@@ -1235,7 +1234,7 @@
;; so the table→transform dep is visible in the dependency graph.
(t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})
(deps.hooks/after-transform-run referenced-transform-id referenced-table-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
(is (=? {:data [{:id unreffed-transform-id
:type "transform"
@@ -1261,7 +1260,7 @@
:type :question
:dataset_query native-query}]
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=unreftest")]
(is (=? {:data [{:id unreffed-snippet-id
:type "snippet"
@@ -1281,7 +1280,7 @@
"model" "dashboard"}}]}]}
:content_type "application/json+vnd.prose-mirror"}]
(events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id document-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=unreftest")]
(is (=? {:data [{:id unreffed-dashboard-id
:type "dashboard"
@@ -1300,7 +1299,7 @@
"model" "document"}}]}]}
:content_type "application/json+vnd.prose-mirror"}]
(events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id unreffed-document-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=unreftest")]
(is (=? {:data [{:id unreffed-document-id
:type "document"
@@ -1319,7 +1318,7 @@
:model/Sandbox {sandbox-id :id} {:group_id group-id
:table_id (mt/id :products)
:card_id sandbox-card-id}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=sandbox")]
(is (=? {:data [{:id sandbox-id
:type "sandbox"
@@ -1337,7 +1336,7 @@
:model/Card {unreffed-metric-id :id} {:name "B - Unreferenced Metric - cardtype"
:type :metric
:dataset_query (lib/query mp products)}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "filtering by model only"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=model&query=cardtype")]
(is (=? {:data [{:id unreffed-model-id
@@ -1376,7 +1375,7 @@
:type :question
:archived true
:dataset_query (lib/query mp products)}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=archivedtest")
card-ids (set (map :id (:data response)))]
(is (contains? card-ids unreffed-card-id))
@@ -1388,7 +1387,7 @@
(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}
:model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest"
:archived true}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")
dashboard-ids (set (map :id (:data response)))]
(is (contains? dashboard-ids unreffed-dashboard-id))
@@ -1400,7 +1399,7 @@
(mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}
:model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest"
:archived true}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")
document-ids (set (map :id (:data response)))]
(is (contains? document-ids unreffed-document-id))
@@ -1414,7 +1413,7 @@
:model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest"
:content "WHERE ID > 20"
:archived true}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")
snippet-ids (set (map :id (:data response)))]
(is (contains? snippet-ids unreffed-snippet-id))
@@ -1435,7 +1434,7 @@
:model/Dashboard {regular-dashboard-id :id} {:name "Regular Dashboard - intcontent"}
:model/Dashboard {internal-dashboard-id :id} {:name "Internal Dashboard - intcontent"
:creator_id config/internal-mb-user-id}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "internal-user cards are filtered out"
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced?types=card&query=intcontent")
@@ -1463,7 +1462,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! (t2/select-one :model/Card :id (:id internal-model))))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card)))
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/breaking?types=card&query=intbreaking")
@@ -1485,7 +1484,7 @@
:archived true}]
(events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")
segment-ids (set (map :id (:data response)))]
(is (contains? segment-ids unreffed-segment-id))
@@ -1504,7 +1503,7 @@
:db_id (mt/id)
:active true
:visibility_type "hidden"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")
table-ids (set (map :id (:data response)))]
(is (contains? table-ids active-table-id))
@@ -1519,7 +1518,7 @@
(mt/with-temp [:model/Table {table1-id :id} {:name "Table 1 - unreftest"}
:model/Table {table2-id :id} {:name "Table 2 - unreftest"}
:model/Table {table3-id :id} {:name "Table 3 - unreftest"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? {:data [{:id table1-id} {:id table2-id}]
:total 3
:offset 0
@@ -1568,7 +1567,7 @@
:archived true}]
(events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")
measure-ids (set (map :id (:data response)))]
(is (contains? measure-ids unreffed-measure-id))
@@ -1594,7 +1593,7 @@
:model/Card {card-regular :id} {:name "Card Regular - personalcolltest"
:type :question
:dataset_query (lib/query mp products)}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "include-personal-collections=false (default) excludes cards in personal collections"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=personalcolltest")
card-ids (set (map :id (:data response)))]
@@ -1616,7 +1615,7 @@
:model/Dashboard {dash-in-personal :id} {:name "Dashboard in Personal - personalcolltest"
:collection_id (personal-collection-id user-id)}
:model/Dashboard {dash-regular :id} {:name "Dashboard Regular - personalcolltest"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "include-personal-collections=false (default) excludes dashboards in personal collections"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=personalcolltest")
dashboard-ids (set (map :id (:data response)))]
@@ -1640,7 +1639,7 @@
:model/Document _ {:name "F Document sorttest"}
:model/Segment _ {:name "G Segment sorttest"}
:model/Measure _ {:name "H Measure sorttest"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1705,7 +1704,7 @@
:table_id table1-id}
:model/Measure _ {:name "Measure with Table 2 sorttest"
:table_id table2-id}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1735,7 +1734,7 @@
:collection_id collection1-id}
:model/NativeQuerySnippet _ {:name "SQL snippets sorttest"}
:model/Transform _ {:name "Transforms sorttest"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1765,7 +1764,7 @@
(break-model-card! model-card))
;; Run analysis in a fresh metadata provider cache session to detect the broken reference
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card)))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=brokentest")]
(is (= [(:id model-card)] (mapv :id (:data response)))
@@ -1790,7 +1789,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "filtering by card returns only card sources"
@@ -1825,7 +1824,7 @@
(break-model-card! (t2/select-one :model/Card :id (:id archived-model))))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=archivedbrokentestcard")
@@ -1860,7 +1859,7 @@
(run-analysis-for-card! (:id card)))
(testing "the dependent card's analysis is now broken, attributing the inactive table as the source"
(is (false? (t2/select-one-fn :result :model/AnalysisFinding
- :analyzed_entity_type "card" :analyzed_entity_id (:id card))))
+ :entity_type "card" :entity_id (:id card))))
(is (t2/exists? :model/AnalysisFindingError
:source_entity_type "table" :source_entity_id (:id table)
:analyzed_entity_type "card" :analyzed_entity_id (:id card))))
@@ -1886,7 +1885,7 @@
(break-model-card! model-card))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=multipledependents")
@@ -1915,7 +1914,7 @@
(break-model-card! model-regular))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "include-personal-collections=false (default) excludes source cards in personal collections"
@@ -1948,7 +1947,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(is (=? {:data [{:id (:id model-card-1)}]
@@ -1981,7 +1980,7 @@
(break-model-card! model-card-b))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-a))
(run-analysis-for-card! (:id dependent-card-b)))
(doseq [sort-direction [:asc :desc]]
@@ -2018,7 +2017,7 @@
(break-model-card! model-in-coll2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(doseq [sort-direction [:asc :desc]]
@@ -2055,7 +2054,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1a))
(run-analysis-for-card! (:id dependent-card-1b))
(run-analysis-for-card! (:id dependent-card-2a)))
@@ -2094,7 +2093,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1a))
(run-analysis-for-card! (:id dependent-card-1b))
(run-analysis-for-card! (:id dependent-card-2a)))
@@ -2139,7 +2138,7 @@
(create-dependent! base-card user "Alpha")
(create-dependent! base-card user "Beta")
(create-dependent! base-card user "Gamma")
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "filters by name"
(is (=? [{:data {:name "Alpha"}}]
(get-dependents base-card-id :query "Alpha"))))
@@ -2155,7 +2154,7 @@
(mt/with-temp [:model/Collection {coll-id :id} {:name "SpecialCollection"}]
(create-dependent! base-card user "Card in root")
(create-dependent! base-card user "Card in collection" :collection_id coll-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:data {:name "Card in collection"}}]
(get-dependents base-card-id :query "SpecialCollection")))))))
@@ -2167,7 +2166,7 @@
(create-dependent! base-card user "In Personal" :collection_id (personal-collection-id user-id))
(create-dependent! base-card user "In Sub" :collection_id sub-coll-id)
(create-dependent! base-card user "Regular")
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "default excludes personal collections"
(is (=? [{:data {:name "Regular"}}]
(get-dependents base-card-id))))
@@ -2180,7 +2179,7 @@
(create-dependent! base-card user "C Card")
(create-dependent! base-card user "A Card")
(create-dependent! base-card user "B Card")
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:data {:name "A Card"}} {:data {:name "B Card"}} {:data {:name "C Card"}}]
(get-dependents base-card-id :sort-column :name :sort-direction :asc)))
(is (=? [{:data {:name "C Card"}} {:data {:name "B Card"}} {:data {:name "A Card"}}]
@@ -2195,7 +2194,7 @@
(create-dependent! base-card user "In C" :collection_id coll-c)
(create-dependent! base-card user "In A" :collection_id coll-a)
(create-dependent! base-card user "In B" :collection_id coll-b)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:data {:name "In A"}} {:data {:name "In B"}} {:data {:name "In C"}}]
(get-dependents base-card-id :sort-column :location :sort-direction :asc)))
(is (=? [{:data {:name "In C"}} {:data {:name "In B"}} {:data {:name "In A"}}]
@@ -2207,7 +2206,7 @@
(let [{low-id :id} (create-dependent! base-card user "Low")
{mid-id :id} (create-dependent! base-card user "Mid")
{high-id :id} (create-dependent! base-card user "High")]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(t2/update! :model/Card low-id {:view_count 10})
(t2/update! :model/Card mid-id {:view_count 50})
(t2/update! :model/Card high-id {:view_count 100})
@@ -2226,7 +2225,7 @@
(t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id base-card-id
:row 0 :col 0 :size_x 4 :size_y 4})
(events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(t2/update! :model/Card low-id {:view_count 10})
(t2/update! :model/Card high-id {:view_count 100})
(is (=? [{:data {:view_count 10}} {:data {:view_count 100}} {:data {:view_count 200}}]
@@ -2241,7 +2240,7 @@
(create-dependent! base-card user "A Match")
(create-dependent! base-card user "B Match")
(create-dependent! base-card user "Should not appear")
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}]
(get-dependents base-card-id :query "Match" :sort-column :name :sort-direction :asc))))))
@@ -2270,7 +2269,7 @@
:analyzed_entity_id card-id
:source_entity_type nil
:source_entity_id nil
- :error_type "invalid-query"}])
+ :error_type "syntax-error"}])
(let [result (#'deps.api/node-errors {:card [card-id]})
card-errors (get result [:card card-id])]
(testing "includes errors with visible source"
@@ -2278,7 +2277,7 @@
(testing "excludes errors with archived source"
(is (not (contains? card-errors {:type :missing-column :detail "col2"}))))
(testing "includes errors with nil source"
- (is (contains? card-errors {:type :invalid-query}))))))))
+ (is (contains? card-errors {:type :syntax-error}))))))))
(deftest ^:synchronized node-downstream-errors-filtering-test
(testing "node-downstream-errors filters by analyzed entity visibility"
@@ -2332,7 +2331,7 @@
(break-model-card! model-card-b))
;; Run analysis to detect broken references
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id visible-dep-a))
(run-analysis-for-card! (:id visible-dep-b))
(run-analysis-for-card! (:id archived-dep-a))
@@ -2475,7 +2474,7 @@
(mt/with-temp [:model/Card _ {:name "A Card - unreftest" :archived true}
:model/Card {card2-id :id} {:name "B Card - unreftest"}
:model/Card {card3-id :id} {:name "C Card - unreftest"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "card"
@@ -2506,7 +2505,7 @@
:archived true}]
(doseq [id [card1-id (:id card2) card3-id card4-id card5-id]]
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id id) :user-id (mt/user->id :crowberto)}))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "card"
@@ -2545,7 +2544,7 @@
:type :query
:query {:source-table table-1-id}}}]
(events/publish-event! :event/card-create {:object card :user-id (:creator_id card)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "after creating card that depends on table-1, only table-2 is unused"
(is (= #{table-2-id}
(->> (mt/user-http-request :crowberto :get 200 "table" :unused-only true)
@@ -2575,7 +2574,7 @@
(break-model-card! model-card-1)
(break-model-card! model-card-2))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "returns only broken dependents for the specified source (not other sources)"
@@ -2602,7 +2601,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card-a))
(run-analysis-for-card! (:id dependent-card-b)))
(testing "count matches dependents-with-errors from /graph/breaking"
@@ -2656,7 +2655,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card)))
;; Admin can access
(is (sequential? (mt/user-http-request :crowberto :get 200
@@ -2691,7 +2690,7 @@
(break-model-card! model-card-1))
;; Run analysis for both model-2 and dependent-card
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id model-card-2))
(run-analysis-for-card! (:id dependent-card)))
;; Query /graph/broken for model-1 - should only return model-2, not dependent-card
@@ -2715,7 +2714,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-card)))
(testing "filtering by card type returns the dependent card"
(let [response (mt/user-http-request :crowberto :get 200
@@ -2748,7 +2747,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(run-analysis-for-card! (:id dependent-in-personal))
(run-analysis-for-card! (:id dependent-regular)))
(testing "include-personal-collections=false (default) excludes broken dependents in personal collections"
@@ -2773,7 +2772,7 @@
:owner_user_id (mt/user->id :crowberto)}
:model/Table {table2-id :id} {:name "Email Owned Table - ownertest"
:owner_email "[email protected]"}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "table" :query "ownertest"
@@ -2807,7 +2806,7 @@
:query (lib/query mp products)}
:target {:schema "PUBLIC"
:name "email_owned_transform_table"}}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "transform" :query "ownertest"
@@ -2866,7 +2865,7 @@
:database_id (mt/id)
:dataset_query (delete-venue-query)}]
(mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [dependent-ids (fn []
(->> (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id (mt/id :venues)
@@ -2896,7 +2895,7 @@
:database_id (mt/id)
:dataset_query (delete-venue-query)}]
(mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [unreferenced (fn [query]
(mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced"
:types "action"
@@ -2910,3 +2909,11 @@
(testing "a search for the data-actions root name finds the action"
(is (matches? "Data actions"))
(is (not (matches? "Our analytics"))))))))))
+
+(deftest backfill-status-requires-superuser-test
+ (testing "GET /api/ee/dependencies/backfill-status scans every entity, so only superusers can call it"
+ (mt/with-premium-features #{:dependencies}
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :get 403 "ee/dependencies/backfill-status")))
+ (is (=? {:complete boolean?}
+ (mt/user-http-request :crowberto :get 200 "ee/dependencies/backfill-status"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
deleted file mode 100644
index 9dee6dc38fb..00000000000
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ /dev/null
@@ -1,401 +0,0 @@
-(ns metabase-enterprise.dependencies.findings-test
- (:require
- [clojure.test :refer :all]
- [medley.core :as m]
- [metabase-enterprise.dependencies.analysis :as deps.analysis]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
- [metabase-enterprise.dependencies.models.analysis-finding-error :as models.analysis-finding-error]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.test-metadata :as meta]
- [metabase.lib.test-util :as lib.tu]
- [metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]
- [toucan2.core :as t2]))
-
-(use-fixtures :once (fixtures/initialize :db))
-
-(defn- backfill-all-entity-analyses! []
- (doseq [model [:card :transform :segment]]
- (while (pos? (deps.findings/analyze-batch! model 100)))))
-
-(deftest ^:synchronized can-analyze-entity-batches-test
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]
- (is (= 2 (deps.findings/analyze-batch! :card 2)))
- (is (= #{models.analysis-finding/*current-analysis-finding-version*}
- (t2/select-fn-set :analysis_version
- :model/AnalysisFinding
- :analyzed_entity_id [:in [card-id other-card-id]]
- :analyzed_entity_type :card)))))))
-
-(deftest ^:synchronized does-not-repeatedly-analyze-entities-test
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}
- :model/Card _ {:dataset_query (lib/query mp orders)}]
- (is (= 2 (deps.findings/analyze-batch! :card 2)))
- (is (= 0 (deps.findings/analyze-batch! :card 2)))))))
-
-(deftest ^:synchronized re-analyze-entities-when-analysis-version-bumped-test
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))
- orders (lib.metadata/table mp (mt/id :orders))]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}
- :model/Card _ {:dataset_query (lib/query mp orders)}]
- (is (= 2 (deps.findings/analyze-batch! :card 2)))
- (is (= 0 (deps.findings/analyze-batch! :card 2)))
- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
- (is (= 2 (deps.findings/analyze-batch! :card 2))))))))
-
-(deftest ^:synchronized does-analyze-native-entities-test
- (testing "failed analysis result is appropriately stored in the appdb"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]
- (is (= 1 (deps.findings/analyze-batch! :card 1)))
- (is (= [models.analysis-finding/*current-analysis-finding-version* false]
- (t2/select-one-fn (juxt :analysis_version :result)
- :model/AnalysisFinding
- :analyzed_entity_id card-id
- :analyzed_entity_type :card))))))))
-
-(deftest ^:synchronized does-report-errors-for-missing-refs-in-fields-test
- (testing "missing (not inactive) field refs in :fields are reported as findings (GHY-3157)"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- orders (lib.metadata/table mp (mt/id :orders))
- base (lib/query mp orders)
- cols (vec (take 5 (lib/returned-columns base orders)))
- bad-col (-> (first cols)
- lib/ref
- (assoc 2 "bad_column"))
- query (lib/with-fields base (conj cols bad-col))]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
- (is (= 1 (deps.findings/analyze-batch! :card 1)))
- (is (= [models.analysis-finding/*current-analysis-finding-version* false]
- (t2/select-one-fn (juxt :analysis_version :result)
- :model/AnalysisFinding
- :analyzed_entity_id card-id
- :analyzed_entity_type :card))))))))
-
-;; TODO: (bshepherdson, 2026-02-05) Add a test like does-not-report-errors-in-removable-refs-test-1-stage-fields for
-;; join clause :fields as well. See QUE-3081 and QUE-3044.
-
-(deftest ^:parallel reports-missing-field-in-downstream-card-fields-test
- (testing "Q2 based on Q1 with :fields referencing a column Q1 no longer returns should be reported (GHY-3157)"
- (let [mp meta/metadata-provider
- q1 (lib/query mp (meta/table-metadata :orders))
- q1-cols (lib/returned-columns q1)
- removed-col (m/find-first #(= (:name %) "ID") q1-cols)
- ;; Card 101 = Q1 with ID removed from result-metadata (simulates Q1 dropping that field)
- mp (lib.tu/metadata-provider-with-card-from-query
- mp 101 q1
- {:result-metadata (vec (remove #(= (:name %) "ID") q1-cols))})
- ;; Q2: query based on card 101, with :fields including the now-missing ID
- q2-base (lib/query mp {:lib/type :metadata/card :id 101})
- q2-cols (lib/returned-columns q2-base)
- q2 (lib/with-fields q2-base (conj (vec q2-cols) (lib/ref removed-col)))
- ;; Card 102 = Q2 (provide result-metadata explicitly since q2 has a bad ref)
- mp (lib.tu/metadata-provider-with-card-from-query
- mp 102 q2
- {:result-metadata (vec q2-cols)})
- results (deps.analysis/check-entity mp :card 102)]
- (is (seq results)
- "Q2 should have findings for the missing ID column from Q1"))))
-
-(defn- stale-map
- "Returns a map of {entity-id stale?} for the given card IDs."
- [card-ids]
- (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id [:in card-ids]))
-
-(defn- finding-stale?
- "Returns the stale value for a specific entity's analysis finding, or nil if no finding exists."
- [entity-type entity-id]
- (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type entity-type
- :analyzed_entity_id entity-id))
-
-(deftest ^:synchronized analyze-batch-picks-up-missing-analyses-test
- (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
- (testing "card has no analysis finding initially"
- (is (not (t2/exists? :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))
- (testing "analyze-batch! creates analysis for the card"
- (lib-be/with-metadata-provider-cache
- (is (pos? (deps.findings/analyze-batch! :card 10)))))
- (testing "card now has an analysis finding"
- (is (t2/exists? :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)))))))))
-
-(deftest mark-dependents-stale-test
- (testing "mark-transitive-dependents-stale! marks direct dependents as stale"
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
- :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
- :to_entity_type :card
- :to_entity_id parent-card-id}]
- (run! deps.findings/upsert-analysis!
- (t2/select :model/Card :id [:in [parent-card-id child-card-id]]))
- (is (= {parent-card-id false, child-card-id false}
- (stale-map [parent-card-id child-card-id]))
- "neither should be stale before marking")
- (t2/with-transaction [_conn]
- (deps.findings/mark-transitive-dependents-stale! {:card [parent-card-id]})
- (is (= {parent-card-id false, child-card-id true}
- (stale-map [parent-card-id child-card-id]))
- "parent should NOT be stale, child should be stale")))))))))
-
-(deftest mark-dependents-stale-transitive-test
- (testing "mark-transitive-dependents-stale! marks transitive dependents as stale"
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {grandparent-id :id} {:dataset_query (lib/query mp products)}
- :model/Card {parent-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp grandparent-id))}
- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id parent-id
- :to_entity_type :card
- :to_entity_id grandparent-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}]
- (run! deps.findings/upsert-analysis!
- (t2/select :model/Card :id [:in [grandparent-id parent-id child-id]]))
- (t2/with-transaction [_conn]
- (deps.findings/mark-transitive-dependents-stale! {:card [grandparent-id]})
- (is (= {grandparent-id false, parent-id true, child-id true}
- (stale-map [grandparent-id parent-id child-id]))
- "grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))
-
-(deftest ^:synchronized mark-subtree-stale-then-drain-test
- (testing "marking an entity stale also marks its whole transitive subtree up front, and check-entities! drains it"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {gp-id :id :as gp-card} {:dataset_query (lib/query mp products)}
- :model/Card {p-id :id :as p-card} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}
- :model/Card {c-id :id :as c-card} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}
- :model/Dependency _ {:from_entity_type :card :from_entity_id p-id
- :to_entity_type :card :to_entity_id gp-id}
- :model/Dependency _ {:from_entity_type :card :from_entity_id c-id
- :to_entity_type :card :to_entity_id p-id}]
- ;; Analyze all three so they have findings
- (run! deps.findings/upsert-analysis! [gp-card p-card c-card])
- (is (= {gp-id false, p-id false, c-id false}
- (stale-map [gp-id p-id c-id]))
- "all should start as not stale")
- ;; A change to the grandparent marks it AND its whole transitive subtree (parent and child) stale up front
- (deps.findings/mark-entity-and-transitive-dependents-stale! :card gp-id)
- (is (= {gp-id true, p-id true, c-id true}
- (stale-map [gp-id p-id c-id]))
- "the grandparent and its whole subtree should be stale")
- ;; The drain analyzes each once, without re-marking anything stale during draining
- (#'task.entity-check/check-entities!)
- (is (= {gp-id false, p-id false, c-id false}
- (stale-map [gp-id p-id c-id]))
- "after check-entities!, the whole subtree should be re-analyzed (not stale)"))))))))
-
-(deftest ^:synchronized transitive-marking-through-non-analyzable-entity-test
- (testing "marking dependents stale reaches analyzable entities beyond non-analyzable intermediaries (e.g., tables)"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))
- products-id (mt/id :products)]
- (mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (mt/with-model-cleanup [:model/AnalysisFinding]
- ;; Chain: card → (depends on) table → (depends on) transform.
- ;; The table is not analyzable, but the card beyond it should still be reached.
- (mt/with-temp [:model/Transform {transform-id :id :as transform}
- {:source {:type :query :query (lib/query mp products)}
- :name "test_transitive_transform"
- :target {:schema "public" :name "transitive_test_table" :type :table}}
- :model/Card {card-id :id :as card}
- {:dataset_query (lib/query mp products)}
- ;; table depends on transform (downstream of transform)
- :model/Dependency _ {:from_entity_type :table :from_entity_id products-id
- :to_entity_type :transform :to_entity_id transform-id}
- ;; card depends on the same table
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id products-id}]
- ;; Analyze the transform and card (tables aren't analyzable)
- (deps.findings/upsert-analysis! transform)
- (deps.findings/upsert-analysis! card)
- (is (false? (finding-stale? :card card-id))
- "card should start as not stale")
- ;; The card is two hops downstream of the transform: transform → table → card
- (deps.findings/mark-transitive-dependents-stale! {:transform [transform-id]})
- (is (true? (finding-stale? :card card-id))
- "card should be stale — transitive marking reached it via the non-analyzable table")
- ;; Run entity-check to drain stale entities
- (#'task.entity-check/check-entities!)
- (is (false? (finding-stale? :card card-id))
- "card should be re-analyzed after entity-check"))))))))
-
-(deftest ^:synchronized cyclic-dependency-drain-terminates-test
- (testing "check-entities! terminates on a cyclic dependency graph — staleness is fixed up front and never re-marked
- during the drain"
- (backfill-all-entity-analyses!)
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {a-id :id :as a-card} {:dataset_query (lib/query mp products)}
- :model/Card {b-id :id :as b-card} {:dataset_query (lib/query mp products)}
- ;; cycle: a depends on b, and b depends on a
- :model/Dependency _ {:from_entity_type :card :from_entity_id a-id
- :to_entity_type :card :to_entity_id b-id}
- :model/Dependency _ {:from_entity_type :card :from_entity_id b-id
- :to_entity_type :card :to_entity_id a-id}]
- (run! deps.findings/upsert-analysis! [a-card b-card])
- (deps.findings/mark-entity-and-transitive-dependents-stale! :card a-id)
- (is (= {a-id true, b-id true}
- (stale-map [a-id b-id]))
- "both cycle members should be marked stale")
- ;; Under the old wave-propagation design this drain would loop forever; bound it so a regression fails fast.
- (let [done (future (#'task.entity-check/check-entities!))]
- (is (not= ::timeout (deref done 30000 ::timeout))
- "check-entities! must terminate on a cyclic graph")
- (is (= {a-id false, b-id false}
- (stale-map [a-id b-id]))
- "both cycle members should be drained (re-analyzed, not stale)")))))))))
-
-(deftest ^:synchronized python-transform-records-clean-finding-test
- (testing "GHY-3584: a Python transform has no query, so upsert-analysis! resolves its database from source_database_id and records a clean finding"
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]
- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :python
- :source-database (mt/id)
- :source-tables []
- :body "def transform():\n pass"}}]
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! (t2/select-one :model/Transform transform-id)))
- (is (=? {:result true :stale false}
- (t2/select-one :model/AnalysisFinding :analyzed_entity_type :transform :analyzed_entity_id transform-id)))
- (is (empty? (models.analysis-finding-error/errors-for-entity :transform transform-id))))))))
-
-(deftest ^:synchronized unanalyzable-entity-records-terminal-error-test
- (testing "A stale entity whose database can't be resolved records a terminal error finding instead of no-oping forever."
- ;; Without the fix, instance-db-id -> nil makes upsert-analysis! no-op: the stale flag is never
- ;; cleared, so the entity is re-selected and re-attempted on every run. The fix records a terminal
- ;; error, clearing the flag.
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]
- (mt/with-temp [:model/Card card {:dataset_query (mt/native-query {:query "SELECT 1"})
- :database_id (mt/id)}]
- (let [cid (:id card)]
- ;; seed a clean finding, mark it stale, then make its database unresolvable
- (models.analysis-finding/upsert-analysis! :card cid true [])
- (models.analysis-finding/mark-stale! {:card #{cid}})
- (with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}
- (fn []
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))))
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id cid))
- "the stale flag is cleared, so the entity stops being re-selected forever")
- (is (seq (models.analysis-finding-error/errors-for-entity :card cid))
- "an error finding is recorded explaining why the entity couldn't be analyzed")))))))
-
-(deftest ^:synchronized unanalyzable-entity-rechecks-when-database-resolvable-again-test
- (testing "After a no-DB terminal error, the entity is analyzed normally once its database resolves again."
- ;; The terminal error finding is not sticky: it clears the stale flag but does not block future
- ;; analysis. When the entity is next marked stale and its database now resolves, upsert-analysis!
- ;; takes the normal path and replaces the terminal error with a real analysis result.
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]
- (let [mp (mt/metadata-provider)
- products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Card card {:dataset_query (lib/query mp products)}]
- (let [cid (:id card)]
- ;; phase 1: database unresolvable -> terminal error recorded, stale cleared
- (with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}
- (fn []
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))))
- (is (seq (models.analysis-finding-error/errors-for-entity :card cid))
- "phase 1: a terminal error is recorded while the database is unresolvable")
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id cid))
- "phase 1: stale flag cleared")
- ;; phase 2: database resolvable again; re-trigger analysis (as an event/job would)
- (models.analysis-finding/mark-stale! {:card #{cid}})
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))
- (is (empty? (models.analysis-finding-error/errors-for-entity :card cid))
- "phase 2: re-analyzed normally — the terminal db error is replaced by a clean result")
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id cid))
- "phase 2: re-analysis cleared stale again"))))))))
-
-(deftest ^:synchronized analyze-batch-shares-one-metadata-provider-test
- (testing "Every entity in an analysis batch must share one MetadataProvider per database. Reading an entity attaches
- a provider to its query, so selecting the batch outside the cache gives each entity a private one. Nothing
- reads through those providers today -- upsert-analysis! builds its own from the database id and passes it
- explicitly -- so this is a latent trap rather than a live leak: the moment an entity is analyzed through its
- own query, a batch of 500 retains 500 providers' worth of fetched metadata. upsert-analysis! already throws
- when no cache is bound, so the cache is meant to cover this work; only the select sat outside it."
- (backfill-all-entity-analyses!)
- (let [providers (atom [])
- native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}
- :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]
- (let [wanted #{card1-id card2-id}
- upsert-analysis! (mt/original-fn #'deps.findings/upsert-analysis!)]
- (mt/with-dynamic-fn-redefs [deps.findings/upsert-analysis!
- (fn [instance]
- (when (wanted (:id instance))
- (swap! providers conj (:lib/metadata (:dataset_query instance))))
- (upsert-analysis! instance))]
- (deps.findings/analyze-batch! :card 500)))
- (testing "both cards were selected, so the assertion below is meaningful"
- (is (= 2 (count @providers))))
- (testing "and they carry the same provider instance"
- ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be
- ;; satisfied by two separate caches and would not detect the regression.
- (is (every? #(identical? (first @providers) %) @providers))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
index 15586d21787..c731c2d78fb 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
@@ -1,16 +1,15 @@
(ns metabase-enterprise.dependencies.hooks-test
(:require
[clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.hooks]
- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.task.backfill :as task.backfill]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.load :as serdes.load]
[metabase.batch-processing.core :as grouper]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -18,173 +17,357 @@
(comment
metabase-enterprise.dependencies.hooks/keep-me)
-(defn- stale? [entity-type entity-id]
- (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true))
+(def ^:private table-query deps.test/table-query)
-(defn- table-query [table]
- (let [mp (mt/metadata-provider)]
- (lib/query mp (lib.metadata/table mp (mt/id table)))))
+(def ^:private card-query deps.test/card-query)
-(defn- card-query [card-id]
- (let [mp (mt/metadata-provider)]
- (lib/query mp (lib.metadata/card mp card-id))))
+(def ^:private pending? deps.test/dependency-pending?)
-(deftest card-update-test
- (testing "updating a card marks it stale only when a column its dependencies come from changes"
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/DependencyStatus]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
- (deps.dependency-status/upsert-status! :card card-id false)
- (t2/update! :model/Card card-id {:name "Renamed"})
- (is (not (stale? :card card-id)))
- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
- (is (stale? :card card-id)))))))
-
-(deftest card-update-marks-finding-test
- (testing "updating a card's query marks its own analysis finding stale in the same transaction"
+(def ^:private computed! deps.test/record-dependencies-computed!)
+
+(defn- status-row-count []
+ (reduce + (map t2/count [:model/DependencyStatus :model/ResultMetadataStatus :model/AnalysisFindingStatus])))
+
+(deftest card-update-adds-no-status-test
+ (testing "updating a card adds no status row, and triggers the jobs once it commits"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/DependencyStatus :model/AnalysisFinding]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
- (deps.analysis-finding/upsert-analysis! :card card-id true [])
- (let [finding-stale? #(t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id)]
- (t2/update! :model/Card card-id {:name "Renamed"})
- (is (not (finding-stale?)))
- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
- (is (finding-stale?))))))))
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (let [submitted (atom [])
+ before (status-row-count)]
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (deps.test/do-with-immediate-hook-writes
+ #(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})))
+ (is (= before (status-row-count)))
+ (is (= [{:dependency-jobs true}] @submitted)))))))
(deftest card-update-without-feature-test
- (testing "without the dependencies feature the hooks do nothing"
+ (testing "without the dependencies feature the hooks trigger nothing"
(mt/with-premium-features #{}
- (mt/with-model-cleanup [:model/DependencyStatus]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
- (deps.dependency-status/upsert-status! :card card-id false)
- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
- (is (not (stale? :card card-id))))))))
-
-(deftest dashboard-card-test
- (testing "inserting, updating and deleting a dashboard card, or adding a series, marks its dashboard stale"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (let [submitted (atom [])]
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (deps.test/do-with-immediate-hook-writes
+ #(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})))
+ (is (= [] @submitted)))))))
+
+(deftest update-marking-nothing-triggers-nothing-test
+ (testing "an update that marks nothing triggers no job, as a sync storing only a field's fingerprint"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Field {field-id :id} {:table_id (mt/id :venues) :name "x"}]
+ (let [submitted (atom [])]
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (deps.test/do-with-immediate-hook-writes
+ #(t2/update! :model/Field field-id {:last_analyzed (t/offset-date-time)})))
+ (is (= [] @submitted)))))))
+
+(deftest output-columns-mark-dependents-test
+ (testing "a metric rename, a field position and a table schema reach the cards reading them, a question rename and a
+ table caveat don't"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}
+ :model/Card {metric-id :id} {:type :metric
+ :dataset_query (lib/aggregate (table-query :venues) (lib/count))}]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :card :to_entity_id source-id})
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :card :to_entity_id metric-id})
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id source-id
+ :to_entity_type :table :to_entity_id (mt/id :venues)})
+ (let [marks? (fn [card write!]
+ (deps.test/record-result-metadata-processed! card)
+ (write!)
+ (deps.test/result-metadata-pending? card))]
+ (is (not (marks? card-id #(t2/update! :model/Card source-id {:name "Renamed"}))))
+ (is (marks? card-id #(t2/update! :model/Card metric-id {:name "Renamed"})))
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:position 5}
+ (is (marks? source-id #(t2/update! :model/Field (mt/id :venues :price) {:position 0}))))
+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:schema "PUBLIC"}
+ (is (marks? source-id #(t2/update! :model/Table (mt/id :venues) {:schema "OTHER"}))))
+ (mt/with-temp [:model/TableUserSettings _ {:table_id (mt/id :venues)}]
+ (is (not (marks? source-id #(t2/update! :model/TableUserSettings (mt/id :venues)
+ {:caveats "Careful"}))))))))))
+
+(deftest card-metadata-dependents-see-test
+ (testing "a card's result metadata change marks its dependents only when it changes what they see"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :card :to_entity_id source-id})
+ (let [metadata (t2/select-one-fn :result_metadata :model/Card :id source-id)
+ marks? (fn [f]
+ (deps.test/record-result-metadata-processed! card-id)
+ (t2/update! :model/Card source-id {:result_metadata (mapv f metadata)})
+ (deps.test/result-metadata-pending? card-id))]
+ (is (not (marks? #(assoc % :fingerprint {:global {:distinct-count 3}}))))
+ (is (marks? #(cond-> % (= "NAME" (:name %)) (assoc :display_name "Renamed")))))))))
+
+(defn- with-card-and-dependent!
+ "Call `f` with a card on `table-key` and a card on that card, whose result metadata and analysis are current, and a
+ function telling whether both are pending for both jobs."
+ [table-key f]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query table-key)}
+ :model/Card {dependent-id :id} {:dataset_query (card-query card-id)}]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id table-key)})
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id dependent-id
+ :to_entity_type :card :to_entity_id card-id})
+ (f (fn []
+ (doseq [id [card-id dependent-id]]
+ (deps.test/record-result-metadata-processed! id)
+ (deps.test/record-analysis-finding-processed! :card id)))
+ (fn []
+ (every? (fn [id] (and (deps.test/result-metadata-pending? id)
+ (deps.test/analysis-finding-pending? :card id)))
+ [card-id dependent-id])))))
+
+(deftest remap-marks-transitive-dependents-test
+ (testing "an external remap, its human-readable field and that field's user settings mark every card that carries
+ the remapped column"
+ (mt/with-premium-features #{:dependencies}
+ (with-card-and-dependent!
+ :orders
+ (fn [processed! pending?]
+ (processed!)
+ (mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :product_id)
+ :type :external
+ :name "Product"
+ :human_readable_field_id (mt/id :products :title)}]
+ (testing "adding the remap"
+ (is (pending?)))
+ (testing "changing its human-readable field"
+ (processed!)
+ (mt/with-temp-vals-in-db :model/Field (mt/id :products :title) {:display_name "Renamed"}
+ (is (pending?))))
+ (testing "a user setting of its human-readable field"
+ (processed!)
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :title) :display_name "Item"}]
+ (is (pending?))))))))))
+
+(deftest internal-remap-values-mark-transitive-dependents-test
+ (testing "the FieldValues of an internally remapped field mark every card carrying the remapped column when they gain
+ or lose human-readable values, and not when only their values change"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/DependencyStatus]
+ (with-card-and-dependent!
+ :venues
+ (fn [processed! pending?]
+ (mt/with-temp [:model/Dimension _ {:field_id (mt/id :venues :price) :type :internal :name "Price"}
+ :model/FieldValues fv {:field_id (mt/id :venues :price) :type :full
+ :values [1 2 3 4] :human_readable_values []}]
+ (processed!)
+ (t2/update! :model/FieldValues (:id fv) {:values [1 2 3 4 5]})
+ (is (not (pending?)))
+ (t2/update! :model/FieldValues (:id fv) {:human_readable_values ["$" "$$" "$$$" "$$$$" "$$$$$"]})
+ (is (pending?))))))))
+
+(deftest batched-marks-test
+ (testing "a batch of field writes marks the cards reading their table once per status, and triggers the jobs once"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :venues)})
+ (deps.test/record-result-metadata-processed! card-id)
+ (let [marks (atom [])
+ submitted (atom [])
+ mark! (mt/original-fn #'dependencies.db/mark-dependents-result-metadata-statuses-stale!)]
+ (mt/with-dynamic-fn-redefs [dependencies.db/mark-dependents-result-metadata-statuses-stale!
+ (fn [entity-type entity-ids]
+ (swap! marks conj [entity-type entity-ids])
+ (mark! entity-type entity-ids))
+ grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:position 5}
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :name) {:position 6}
+ (reset! marks [])
+ (reset! submitted [])
+ (deps.test/do-with-immediate-hook-writes
+ #(dependencies.hooks/with-batched-marks
+ (t2/update! :model/Field (mt/id :venues :price) {:position 0})
+ (t2/update! :model/Field (mt/id :venues :name) {:position 1})))
+ (is (= [[:table #{(mt/id :venues)}]] @marks))
+ (is (= [{:dependency-jobs true}] @submitted))
+ (is (deps.test/result-metadata-pending? card-id))))))))))
+
+(deftest snippet-delete-marks-transitive-dependencies-test
+ (testing "deleting a snippet nested in another marks the dependencies of the cards expanding it"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner delete" :content "VENUES"}
+ :model/NativeQuerySnippet {outer-id :id} {:name "outer delete" :content "{{snippet: inner delete}}"}
+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "{{snippet: outer delete}}")}]
+ (t2/insert! :model/Dependency {:from_entity_type :snippet :from_entity_id outer-id
+ :to_entity_type :snippet :to_entity_id inner-id})
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :snippet :to_entity_id outer-id})
+ (computed! :card card-id)
+ (t2/delete! :model/NativeQuerySnippet :id inner-id)
+ (is (pending? :card card-id))))))
+
+(deftest card-marks-own-statuses-test
+ (testing "an update of what a job reads of a card marks the card's own status for that job stale, and a question's
+ rename marks nothing"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (computed! :card card-id)
+ (t2/update! :model/Card card-id {:name "Renamed"})
+ (is (not (pending? :card card-id)))
+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
+ (is (pending? :card card-id))))))
+
+(deftest dashboard-parts-mark-dashboard-test
+ (testing "inserting, updating and deleting a dashboard card, a series or a tab marks the dashboard's dependencies stale"
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Dashboard {dashboard-id :id} {}
:model/Card {card-id :id} {:dataset_query (table-query :products)}
:model/Card {other-id :id} {:dataset_query (table-query :orders)}]
- (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id false)]
- (processed!)
- (let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard
- {:dashboard_id dashboard-id :card_id card-id
- :row 0 :col 0 :size_x 4 :size_y 4})]
- (testing "inserting a dashboard card"
- (is (stale? :dashboard dashboard-id)))
- (processed!)
- (t2/update! :model/DashboardCard dashcard-id {:row 1})
- (testing "moving a dashboard card"
- (is (not (stale? :dashboard dashboard-id))))
- (t2/update! :model/DashboardCard dashcard-id {:card_id other-id})
- (testing "pointing a dashboard card at another card"
- (is (stale? :dashboard dashboard-id)))
- (processed!)
- (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dashcard-id :card_id card-id :position 0})
- (testing "adding a series"
- (is (stale? :dashboard dashboard-id)))
- (processed!)
- (t2/delete! :model/DashboardCard :id dashcard-id)
- (testing "removing a dashboard card"
- (is (stale? :dashboard dashboard-id))))))))))
+ (let [changes? (fn [write!]
+ (computed! :dashboard dashboard-id)
+ (write!)
+ (pending? :dashboard dashboard-id))
+ dashcard (atom nil)]
+ (testing "inserting a dashboard card"
+ (is (changes? #(reset! dashcard (t2/insert-returning-pk! :model/DashboardCard
+ {:dashboard_id dashboard-id :card_id card-id
+ :row 0 :col 0 :size_x 4 :size_y 4})))))
+ (testing "pointing a dashboard card at another card"
+ (is (changes? #(t2/update! :model/DashboardCard @dashcard {:card_id other-id}))))
+ (testing "adding a series"
+ (is (changes? #(t2/insert! :model/DashboardCardSeries {:dashboardcard_id @dashcard :card_id card-id
+ :position 0}))))
+ (testing "removing a series"
+ (is (changes? #(t2/delete! :model/DashboardCardSeries :dashboardcard_id @dashcard))))
+ (testing "adding and removing a tab"
+ (let [tab-id (atom nil)]
+ (is (changes? #(reset! tab-id (t2/insert-returning-pk! :model/DashboardTab
+ {:dashboard_id dashboard-id :name "Tab"
+ :position 0}))))
+ (is (changes? #(t2/delete! :model/DashboardTab :id @tab-id)))))
+ (testing "removing a dashboard card"
+ (is (changes? #(t2/delete! :model/DashboardCard :id @dashcard))))
+ (testing "nothing else"
+ (computed! :dashboard dashboard-id)
+ (is (not (pending? :dashboard dashboard-id))))))))))
+
+(deftest table-parts-mark-cards-test
+ (testing "a write of a table, its user settings, one of its fields, field user settings or dimensions, marks the cards
+ on it stale"
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :venues)})
+ (let [table-id (mt/id :venues)
+ field-id (mt/id :venues :price)
+ changes? (fn [write!]
+ (deps.test/record-result-metadata-processed! card-id)
+ (write!)
+ (deps.test/result-metadata-pending? card-id))]
+ (mt/with-temp-vals-in-db :model/Table table-id {:display_name "Venues"}
+ (is (changes? #(t2/update! :model/Table table-id {:display_name "Places"}))))
+ (mt/with-temp-vals-in-db :model/Field field-id {:display_name "Price"}
+ (is (changes? #(t2/update! :model/Field field-id {:display_name "Cost"}))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id :display_name "Cost"}]
+ (is (changes? (fn [] (t2/update! :model/FieldUserSettings :field_id field-id {:display_name "Price"})))))
+ (is (changes? #(t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Places"})))
+ (is (changes? #(t2/update! :model/TableUserSettings table-id {:display_name "Spots"})))
+ (is (changes? #(t2/delete! :model/TableUserSettings :table_id table-id)))
+ (let [dimension-id (atom nil)]
+ (is (changes? #(reset! dimension-id (t2/insert-returning-pk! :model/Dimension
+ {:field_id field-id :name "Price"
+ :type :internal}))))
+ (is (changes? #(t2/delete! :model/Dimension :id @dimension-id))))))))))
(deftest entity-delete-test
- (testing "deleting an entity marks its status stale, and the backfill drops its dependencies and status"
+ (testing "the dependency job drops the dependencies and status of a deleted entity"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
(t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id
:to_entity_type "table" :to_entity_id (mt/id :products)})
- (deps.dependency-status/upsert-status! :card card-id false)
+ (computed! :card card-id)
(t2/delete! :model/Card :id card-id)
- (is (stale? :card card-id))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (not (t2/exists? :model/Dependency :from_entity_type "card" :from_entity_id card-id)))
(is (not (t2/exists? :model/DependencyStatus :entity_type "card" :entity_id card-id)))))))
(deftest concurrent-saves-of-dependent-cards-commit-test
(testing "saving a card and then its dependent in one transaction, while another request saves the dependent, commits both"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/DependencyStatus]
- (let [insert-card! (fn [query]
- (t2/insert-returning-pk! :model/Card {:name (mt/random-name)
- :display :table
- :visualization_settings {}
- :creator_id (mt/user->id :crowberto)
- :database_id (mt/id)
- :dataset_query query}))
- card-a (insert-card! (table-query :products))
- card-b (insert-card! (card-query card-a))
- a-saved (promise)
- b-status-marked (promise)
- save! (fn [thunk]
- (future
- (try
- (thunk)
- :committed
- (catch Throwable e
- (ex-message (or (ex-cause e) e))))))
- mark-stale! (mt/original-fn #'deps.dependency-status/mark-stale!)]
+ (mt/with-model-cleanup [:model/Card]
+ (let [insert-card! (fn [query]
+ (t2/insert-returning-pk! :model/Card {:name (mt/random-name)
+ :display :table
+ :visualization_settings {}
+ :creator_id (mt/user->id :crowberto)
+ :database_id (mt/id)
+ :dataset_query query}))
+ card-a (insert-card! (table-query :products))
+ card-b (insert-card! (card-query card-a))
+ a-saved (promise)
+ save! (fn [thunk]
+ (future
+ (try
+ (thunk)
+ :committed
+ (catch Throwable e
+ (ex-message (or (ex-cause e) e))))))]
(t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b
:to_entity_type :card :to_entity_id card-a})
- (doseq [card-id [card-a card-b]]
- (deps.dependency-status/upsert-status! :card card-id false)
- (deps.analysis-finding/upsert-analysis! :card card-id true []))
- (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!
- (fn [type->ids]
- (mark-stale! type->ids)
- (when (= {:card #{card-b}} type->ids)
- (deliver b-status-marked true)))]
- (let [cascade (save! #(t2/with-transaction [_conn]
- (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})
- (deliver a-saved true)
- (deref b-status-marked 10000 nil)
- (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))
- single (save! #(do (deref a-saved 10000 nil)
- (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]
- (is (= [:committed :committed]
- [(deref cascade 30000 :timeout) (deref single 30000 :timeout)])))))))))
-
-(deftest serdes-load-triggers-jobs-once-test
- (testing "a serdes load marks what it writes, nested deletes included, and triggers the jobs once it finishes"
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/DependencyStatus]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}
- :model/Dashboard {dashboard-id :id} {}
- :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]
- (let [exported (ts/extract-one "Dashboard" dashboard-id)
- submitted (atom [])
- triggered (atom 0)]
- (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id
- :row 4 :col 0 :size_x 4 :size_y 4})
- (deps.dependency-status/upsert-status! :dashboard dashboard-id false)
- (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))
- task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))]
- (deps.test/do-with-immediate-hook-writes
- #(serdes/with-cache
- (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
- (testing "deleting the dashboard card missing from the export marks its dashboard"
- (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))
- (is (stale? :dashboard dashboard-id)))
- (testing "the load triggers the jobs once, after it finishes"
- (is (= [] @submitted))
- (is (= 1 @triggered)))))))))
-
-(deftest serdes-reload-of-unchanged-dashboard-test
- (testing "reloading an unchanged dashboard export, including empty visualization settings, does not mark it"
+ (let [cascade (save! #(t2/with-transaction [_conn]
+ (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})
+ (deliver a-saved true)
+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))
+ single (save! #(do (deref a-saved 10000 nil)
+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]
+ (is (= [:committed :committed]
+ [(deref cascade 30000 :timeout) (deref single 30000 :timeout)]))))))))
+
+(deftest serdes-load-triggers-jobs-test
+ (testing "a serdes load triggers the jobs through the writes it makes"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/DependencyStatus]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}
- :model/Dashboard {dashboard-id :id} {}
- :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id
- :visualization_settings {}}]
- (let [exported (ts/extract-one "Dashboard" dashboard-id)]
- (deps.dependency-status/upsert-status! :dashboard dashboard-id false)
- (mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])]
- (deps.test/do-with-immediate-hook-writes
- #(serdes/with-cache
- (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
- (is (not (stale? :dashboard dashboard-id)))))))))
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}
+ :model/Dashboard {dashboard-id :id} {}
+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]
+ (let [exported (ts/extract-one "Dashboard" dashboard-id)
+ submitted (atom [])]
+ (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id
+ :row 4 :col 0 :size_x 4 :size_y 4})
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (deps.test/do-with-immediate-hook-writes
+ #(serdes/with-cache
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
+ (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))
+ (is (seq @submitted)))))))
+
+(deftest ^:synchronized database-delete-marks-source-transforms-stale-test
+ (testing "deleting a database marks its source transforms' analysis findings stale so they surface on /dependency-diagnostics/broken (GDGT-2447)"
+ (deps.test/run-with-dependencies-setup!
+ (fn [_mp]
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Transform {transform-id :id}
+ {:source {:type :query
+ :query {:database db-id
+ :type :native
+ :native {:query "SELECT 1"}}}
+ :target {:schema "public" :name "out" :type :table}}
+ :model/AnalysisFinding _ {:entity_type :transform
+ :entity_id transform-id
+ :result true
+ :analyzed_at :%now}]
+ (t2/delete! :model/Database db-id)
+ (is (nil? (t2/select-one :model/Database db-id))
+ "the Database row is actually gone")
+ (is (pending? :transform transform-id))
+ (deps.test/synchronously-run-dependency-job!)
+ (testing "and the analysis finding job flips :result false so the transform shows on broken-diagnostics"
+ (deps.test/run-analysis-finding-job!)
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_type :transform
+ :entity_id transform-id)))
+ (testing "an analysis_finding_error row points the transform at itself so the diagnostics page (`/graph/breaking`) surfaces it"
+ (is (=? {:source_entity_type :transform
+ :source_entity_id transform-id
+ :error_type :validation-exception-error}
+ (t2/select-one :model/AnalysisFindingError
+ :analyzed_entity_type :transform
+ :analyzed_entity_id transform-id))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
index e223a03d437..919f23bc793 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
@@ -1,76 +1,62 @@
(ns metabase-enterprise.dependencies.models.analysis-finding-test
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.test :as mt]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent CyclicBarrier TimeUnit)))
(set! *warn-on-reflection* true)
-(deftest upsert-analysis-clears-stale-flag-test
- (testing "upsert-analysis! sets stale to false"
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {}]
- (t2/with-transaction [_conn]
- ;; Insert a finding with stale = true
- (t2/insert! :model/AnalysisFinding
- {:analyzed_entity_type :card
- :analyzed_entity_id card-id
- :analysis_version (dec models.analysis-finding/*current-analysis-finding-version*)
- :analyzed_at (java.time.OffsetDateTime/now)
- :result true
- :stale true})
- ;; Verify it's stale
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)))
- ;; Now upsert a new analysis
- (models.analysis-finding/upsert-analysis! :card card-id true [])
- ;; Verify stale is now false
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))))))
+(defn- finding [card-id]
+ (t2/select-one :model/AnalysisFinding :entity_type :card :entity_id card-id))
-(deftest upsert-analysis-new-finding-has-stale-false-test
- (testing "upsert-analysis! creates new findings with stale = false"
+(deftest ^:synchronized take-pending-analysis-finding-statuses-test
+ (testing "the analysis job takes the entities without a status, with one counting a write or one of an older
+ analysis, the last ones last, only once their dependencies are current, and inserts the missing statuses
+ counting one write"
(mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {}]
- (t2/with-transaction [_conn]
- ;; Upsert a new analysis (no existing finding)
- (models.analysis-finding/upsert-analysis! :card card-id true [])
- ;; Verify stale is false
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))))))
+ (mt/with-model-cleanup [:model/AnalysisFindingStatus :model/DependencyStatus]
+ (mt/with-temp [:model/Card {unanalyzed :id} {}
+ :model/Card {current :id} {}
+ :model/Card {outdated :id} {}
+ :model/Card {changed :id} {}
+ :model/Card {blocked :id} {}]
+ (let [ids #{unanalyzed current outdated changed blocked}
+ version models.analysis-finding/*current-analysis-finding-version*
+ selected #(into [] (filter (comp ids :entity_id))
+ (dependencies.db/take-pending-analysis-finding-statuses!
+ :card 100000 version models.dependency/current-dependency-analysis-version))]
+ (doseq [card-id (disj ids blocked)]
+ (deps.test/record-dependencies-computed! :card card-id))
+ (doseq [card-id [current changed]]
+ (deps.test/record-status-processed! :model/AnalysisFindingStatus :card card-id version))
+ (deps.test/record-status-processed! :model/AnalysisFindingStatus :card outdated (dec version))
+ (dependencies.db/mark-analysis-finding-statuses-stale! {:card #{changed}})
+ (let [picked (selected)]
+ (is (= #{{:entity_id unanalyzed :stale 1} {:entity_id changed :stale 1}} (set (butlast picked))))
+ (is (= {:entity_id outdated :stale 0} (last picked)))
+ (is (= 1 (t2/select-one-fn :stale :model/AnalysisFindingStatus :entity_type :card
+ :entity_id unanalyzed))))))))))
-(deftest ^:synchronized instances-for-analysis-serves-oldest-analyzed-first-test
- (testing "Among stale entities, instances-for-analysis returns the oldest-analyzed first."
- ;; Fairness tiebreak: the primary sort only ranks stale-before-outdated, so among equally-stale
- ;; rows the order was DB-arbitrary. A batch smaller than the stale backlog could then keep
- ;; serving the same subset, starving the rest across runs. Ordering oldest-`analyzed_at`-first
- ;; makes selection round-robin: whatever was just attempted has the newest stamp and sinks to the
- ;; back, so every stale entity is reached before any is revisited.
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding]
- (mt/with-temp [:model/Card {c1 :id} {}
- :model/Card {c2 :id} {}
- :model/Card {c3 :id} {}
- :model/Card {c4 :id} {}]
- (let [base (java.time.OffsetDateTime/of 2020 1 1 0 0 0 0 java.time.ZoneOffset/UTC)
- stale! (fn [card-id minutes]
- (t2/insert! :model/AnalysisFinding
- {:analyzed_entity_type :card
- :analyzed_entity_id card-id
- :analysis_version models.analysis-finding/*current-analysis-finding-version*
- :analyzed_at (.plusMinutes base minutes)
- :result true
- :stale true}))]
- ;; Insert newest-analyzed first (c4 -> c1) so the unordered scan order is the OPPOSITE of
- ;; what the tiebreak must produce: without the fix this returns [c4 c3], with it [c1 c2].
- (stale! c4 3)
- (stale! c3 2)
- (stale! c2 1)
- (stale! c1 0)
- (let [batch (models.analysis-finding/instances-for-analysis :card 2)]
- (is (= [c1 c2] (mapv :id batch))
- "a sub-backlog batch serves the two oldest-analyzed stale cards, in ascending order"))))))))
+(deftest ^:synchronized concurrent-upserts-test
+ (testing "two analyses of an entity inserting its finding at once leave the finding they found, not an insert error"
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (let [barrier (CyclicBarrier. 2)
+ insert! (mt/original-fn #'t2/insert-returning-pk!)]
+ (mt/with-dynamic-fn-redefs [t2/insert-returning-pk! (fn [model & args]
+ (when (= :model/AnalysisFinding model)
+ (.await barrier 10 TimeUnit/SECONDS))
+ (apply insert! model args))]
+ (let [upserts (repeatedly 2 #(future (models.analysis-finding/upsert-analysis! :card card-id true #{})))]
+ (run! deref (doall upserts))))
+ (is (=? {:result true} (finding card-id)))
+ (is (zero? (t2/count :model/AnalysisFindingError :analyzed_entity_type "card"
+ :analyzed_entity_id card-id))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index a9a5ae489e8..6324be20e89 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -47,43 +47,44 @@
(deftest ^:synchronized card-deps-maintenance-test-1-new-card
(testing "upstream deps of a card are updated correctly"
- (mt/dataset test-data
- (mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
- (let [card1 (card/create-card! (basic-orders) user)]
- (deps.test/synchronously-run-backfill!)
- (is (integer? (:id card1)))
- (testing "when creating a new card"
- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
- (upstream-of :card (:id card1))))
- (testing "that depends on another card"
- (let [card2 (card/create-card! (wrap-card card1) user)]
- (deps.test/synchronously-run-backfill!)
- (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}
- (upstream-of :card (:id card2))))
- (testing "but that doesn't affect the upstream deps of the inner card"
- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
- (upstream-of :card (:id card1))))))))
- (testing "when updating an existing card"
- (testing "to add a new table dep"
- (card/update-card! {:card-before-update card1
- :card-updates
- {:dataset_query (mt/mbql-query orders
- {:joins [{:alias "Products"
- :source-table (mt/id :products)
- :condition [:= $id &Products.$products.id]}]})}})
- (deps.test/synchronously-run-backfill!)
- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))
- (depends-on-> :card (:id card1) :table (mt/id :products))}
- (upstream-of :card (:id card1)))))
- (testing "to remove a table dep"
- (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))
- :card-updates
- {:dataset_query (mt/mbql-query products)}})
- (deps.test/synchronously-run-backfill!)
- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}
- (upstream-of :card (:id card1)))))))))))))
+ (mt/test-helpers-set-global-values!
+ (mt/dataset test-data
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
+ (let [card1 (card/create-card! (basic-orders) user)]
+ (deps.test/synchronously-run-dependency-job!)
+ (is (integer? (:id card1)))
+ (testing "when creating a new card"
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+ (upstream-of :card (:id card1))))
+ (testing "that depends on another card"
+ (let [card2 (card/create-card! (wrap-card card1) user)]
+ (deps.test/synchronously-run-dependency-job!)
+ (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}
+ (upstream-of :card (:id card2))))
+ (testing "but that doesn't affect the upstream deps of the inner card"
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+ (upstream-of :card (:id card1))))))))
+ (testing "when updating an existing card"
+ (testing "to add a new table dep"
+ (card/update-card! {:card-before-update card1
+ :card-updates
+ {:dataset_query (mt/mbql-query orders
+ {:joins [{:alias "Products"
+ :source-table (mt/id :products)
+ :condition [:= $id &Products.$products.id]}]})}})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))
+ (depends-on-> :card (:id card1) :table (mt/id :products))}
+ (upstream-of :card (:id card1)))))
+ (testing "to remove a table dep"
+ (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))
+ :card-updates
+ {:dataset_query (mt/mbql-query products)}})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}
+ (upstream-of :card (:id card1))))))))))))))
(deftest ^:synchronized card-deps-graph-test-1-mbql-card-chain
(testing "deps graph is connected properly for a chain of MBQL cards"
@@ -94,7 +95,7 @@
(let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
{id2 :id :as card2} (card/create-card! (wrap-card card1) user)
{id3 :id :as card3} (card/create-card! (wrap-card card2) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "raw deps are recorded correctly"
(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
@@ -133,7 +134,7 @@
{id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user)
{id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user)
{id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "raw deps are recorded correctly"
(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
@@ -176,52 +177,52 @@
{:aggregation [[:metric (:id metric-card)]]})
:visualization_settings {}}
user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "raw deps are recorded correctly for question using metric"
(is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))
(depends-on-> :card (:id question-card) :card (:id metric-card))}
(upstream-of :card (:id question-card))))))))))))
-(deftest dependency-status-marked-stale-test
- (testing "dependency_status is marked stale when entities are updated, and not touched when they are created"
- (mt/with-model-cleanup [:model/DependencyStatus]
- (testing "cards"
- (mt/with-model-cleanup [:model/Dependency]
- (mt/with-temp [:model/Card card {}]
- (mt/with-premium-features #{:dependencies}
- (card/update-card! {:card-before-update card
- :card-updates {:dataset_query (mt/mbql-query orders)}}))
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true)))))
- (testing "transforms create"
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Transform transform]
- (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform)))))))
- (testing "transforms update"
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Transform transform]
- (mt/with-premium-features #{:dependencies}
- (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))
- (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))
- (testing "snippets create"
+(def ^:private pending? deps.test/dependency-pending?)
+
+(def ^:private computed! deps.test/record-dependencies-computed!)
+
+(deftest ^:synchronized dependency-status-marks-test
+ (testing "updating what the dependency job reads of an entity marks its status stale, and creating one writes no
+ status"
+ (mt/test-helpers-set-global-values!
+ (mt/with-model-cleanup [:model/DependencyStatus :model/Dependency]
(mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/NativeQuerySnippet snippet]
- (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet)))))))
- (testing "snippets update"
- (mt/with-premium-features #{}
- (mt/with-temp [:model/NativeQuerySnippet snippet]
- (mt/with-premium-features #{:dependencies}
- (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))
- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))
+ (testing "cards"
+ (mt/with-temp [:model/Card card {}]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))
+ (computed! :card (:id card))
+ (card/update-card! {:card-before-update card
+ :card-updates {:dataset_query (mt/mbql-query orders)}})
+ (is (pending? :card (:id card)))))
+ (testing "transforms"
+ (mt/with-temp [:model/Transform transform]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform))))
+ (computed! :transform (:id transform))
+ (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}})
+ (is (pending? :transform (:id transform)))))
+ (testing "snippets"
+ (mt/with-temp [:model/NativeQuerySnippet snippet]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet))))
+ (computed! :snippet (:id snippet))
+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"})
+ (is (pending? :snippet (:id snippet))))))))))
(deftest dependency-status-absent-until-backfill-on-card-create-test
- (testing "a created card has no dependency_status row until the backfill job processes it"
+ (testing "a created card has no dependency_status row until the dependency job processes it"
(mt/with-premium-features #{:dependencies}
(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user]
(let [card (card/create-card! (basic-orders) user)]
(is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))
- (deps.test/synchronously-run-backfill!)
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale false))))))))
+ (deps.test/synchronously-run-dependency-job!)
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card)))
+ (is (not (pending? :card (:id card))))))))))
(deftest filtered-graph-dependencies-test
(testing "filtered-graph-dependencies respects filter clause"
@@ -231,7 +232,7 @@
(let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
{id2 :id :as card2} (card/create-card! (wrap-card card1) user)
{id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "without filter, returns all dependencies"
(let [graph (deps.graph/graph-dependencies)
deps (graph/transitive graph [[:card id3]])]
@@ -258,7 +259,7 @@
(let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
{id2 :id :as card2} (card/create-card! (wrap-card card1) user)
{id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "without filter, returns all dependents"
(let [graph (deps.graph/graph-dependents)
deps (graph/transitive graph [[:card id1]])]
@@ -290,7 +291,7 @@
:to_entity_type :table
:to_entity_id (:id table1)})
(deps.graph/swap-dependency! :card (:id card1) [:table (:id table1)] [:table (:id table2)])
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (not (t2/exists? :model/Dependency
:from_entity_type :card
:from_entity_id (:id card1)
@@ -315,7 +316,7 @@
:to_entity_id (:id table2)}])
;; This would fail with duplicate key error before the fix
(deps.graph/swap-dependency! :card (:id card2) [:table (:id table1)] [:table (:id table2)])
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (not (t2/exists? :model/Dependency
:from_entity_type :card
:from_entity_id (:id card2)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.clj
new file mode 100644
index 00000000000..512a18bfa9c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.clj
@@ -0,0 +1,752 @@
+(ns metabase-enterprise.dependencies.result-metadata-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]
+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
+ [metabase-enterprise.test :as met]
+ [metabase.batch-processing.core :as grouper]
+ [metabase.config.core :as config]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.queries.core :as queries]
+ [metabase.query-processor :as qp]
+ [metabase.query-processor.db :as qp.db]
+ [metabase.test :as mt]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private table-query deps.test/table-query)
+
+(def ^:private card-query deps.test/card-query)
+
+(def ^:private pending? deps.test/result-metadata-pending?)
+
+(defn- do-with-inference
+ "Run `thunk` with the job's inference of a card done by `f`, called with the real inference, the query and the card."
+ [f thunk]
+ (let [infer (mt/original-fn #'queries/infer-metadata-with-model-overrides)]
+ (mt/with-dynamic-fn-redefs [queries/infer-metadata-with-model-overrides (fn [query card & _] (f infer query card))]
+ (thunk))))
+
+(defn- measure-query [measure-id]
+ (let [mp (mt/metadata-provider)]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/aggregate (lib.metadata/measure mp measure-id)))))
+
+(defn- native-card-query [card-id]
+ (let [tag (str "#" card-id)]
+ (-> (lib/native-query (mt/metadata-provider) (str "SELECT * FROM {{" tag "}}"))
+ (lib/with-template-tags {tag {:type :card
+ :card-id card-id
+ :name tag
+ :display-name tag
+ :id (str (random-uuid))}}))))
+
+(defn- run-jobs!
+ "Run the dependency job, whose current dependencies the result metadata job waits for, then the result metadata job,
+ as each triggers the other, until neither has anything left to do."
+ []
+ (mt/with-premium-features #{:dependencies}
+ (loop [runs 1]
+ (deps.test/synchronously-run-dependency-job!)
+ (when (and (pos? (deps.result-metadata/update-result-metadata! 1000)) (< runs 20))
+ (recur (inc runs))))))
+
+(defn- column [card-id column-name]
+ (some #(when (= column-name (:name %)) %)
+ (t2/select-one-fn :result_metadata :model/Card :id card-id)))
+
+(defn- rename-column [metadata column-name display-name]
+ (mapv #(cond-> % (= column-name (:name %)) (assoc :display_name display-name)) metadata))
+
+(defmacro ^:private with-dependencies-feature
+ "Run `body` with the dependencies feature, each write committing on its own as it does between real requests."
+ [& body]
+ `(mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ ~@body)))
+
+(defn- reprocess!
+ "Have the result metadata job process the Cards with `card-ids` again, as if they changed."
+ [& card-ids]
+ (dependencies.db/delete-result-metadata-statuses! (vec card-ids)))
+
+(defn- rederive!
+ "Run one batch of the result metadata job."
+ []
+ (mt/with-premium-features #{:dependencies}
+ (deps.result-metadata/update-result-metadata! 1000)))
+
+(defn- add-grand-total [query]
+ (lib/expression query "Grand total" (lib/* (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total)) 2)))
+
+(deftest model-query-edit-test
+ (testing "editing a model's query re-derives the MBQL cards depending on it transitively, but not past a native card"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}
+ :model/Card {question-id :id} {:dataset_query (card-query model-id)}
+ :model/Card {nested-id :id} {:dataset_query (card-query question-id)}
+ :model/Card {native-id :id} {:dataset_query (native-card-query model-id)}
+ :model/Card {after-native-id :id} {:dataset_query (card-query native-id)}]
+ (run-jobs!)
+ (is (nil? (column question-id "Grand total")))
+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})
+ (is (pending? question-id))
+ (run-jobs!)
+ (is (some? (column question-id "Grand total")))
+ (is (some? (column nested-id "Grand total")))
+ (testing "the card depending on the native card is left alone"
+ (is (nil? (column after-native-id "Grand total"))))
+ (is (not-any? pending? [model-id question-id nested-id]))
+ (testing "a column no longer inferred is removed"
+ (t2/update! :model/Card model-id {:dataset_query (table-query :orders)})
+ (run-jobs!)
+ (is (nil? (column question-id "Grand total")))
+ (is (nil? (column nested-id "Grand total"))))))))
+
+(deftest run-metadata-test
+ (testing "re-deriving a card keeps the fingerprints and types its run stored for the columns it still has"
+ (with-dependencies-feature
+ (let [mp (mt/metadata-provider)
+ query (-> (table-query :orders)
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib.metadata/field mp (mt/id :orders :product_id))))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
+ (mt/with-test-user :crowberto
+ (qp/process-query (qp/userland-query query {:card-id card-id})))
+ (let [run-count (column card-id "count")]
+ (is (some? (:fingerprint run-count)))
+ (run-jobs!)
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :product_id) :display_name "Item"}]
+ (run-jobs!)
+ (is (= "Item" (:display_name (column card-id "PRODUCT_ID"))))
+ (is (= (select-keys run-count [:base_type :fingerprint])
+ (select-keys (column card-id "count") [:base_type :fingerprint]))))))))))
+
+(deftest fingerprints-come-only-from-stored-metadata-test
+ (testing "re-deriving a card keeps the stored fingerprints and adds none for a column the stored metadata lacks"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (let [marker {:global {:distinct-count 7}}
+ stored (->> (t2/select-one-fn :result_metadata :model/Card :id card-id)
+ (remove #(= "PRICE" (:name %)))
+ (mapv #(cond-> % (= "ID" (:name %)) (assoc :fingerprint marker))))]
+ (t2/update! :model/Card card-id {:result_metadata stored})
+ (run-jobs!)
+ (reprocess! card-id)
+ (run-jobs!)
+ (is (some? (column card-id "PRICE")))
+ (is (nil? (:fingerprint (column card-id "PRICE"))))
+ (is (= marker (:fingerprint (column card-id "ID")))))))))
+
+(deftest native-parent-test
+ (testing "a card depending on a changed model directly is re-derived even when it also depends on a native card"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}
+ :model/Card {native-id :id} {:dataset_query (native-card-query model-id)
+ :result_metadata (t2/select-one-fn :result_metadata :model/Card
+ :id model-id)}]
+ (let [mp (mt/metadata-provider)
+ query (card-query model-id)
+ native (lib.metadata/card mp native-id)
+ joined (lib/join query (lib/join-clause native [(lib/= (lib/ref (lib.metadata/field mp (mt/id :orders :id)))
+ (lib/ref (lib.metadata/field mp (mt/id :orders :id))))]))]
+ (mt/with-temp [:model/Card {joined-id :id} {:dataset_query joined}]
+ (run-jobs!)
+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})
+ (run-jobs!)
+ (is (some? (column joined-id "Grand total")))))))))
+
+(deftest internal-card-test
+ (testing "the job leaves the result metadata of cards Metabase created itself, such as Usage Analytics, alone"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query model-id)}
+ :model/Card {internal-id :id} {:dataset_query (card-query model-id)
+ :creator_id config/internal-mb-user-id}]
+ (run-jobs!)
+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})
+ (run-jobs!)
+ (is (some? (column card-id "Grand total")))
+ (is (nil? (column internal-id "Grand total")))))))
+
+(deftest metadata-edit-test
+ (testing "editing a model's column overrides re-derives the cards on the model, which take them"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query model-id)}]
+ (run-jobs!)
+ (t2/update! :model/Card model-id {:result_metadata (rename-column (t2/select-one-fn :result_metadata :model/Card
+ :id model-id)
+ "TOTAL" "Grand total")})
+ (is (pending? card-id))
+ (run-jobs!)
+ (is (= "Grand total" (:display_name (column card-id "TOTAL"))))))))
+
+(deftest model-overrides-test
+ (testing "re-deriving a model after its table changed keeps the model's column overrides"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (t2/update! :model/Card model-id {:result_metadata (rename-column (t2/select-one-fn :result_metadata :model/Card
+ :id model-id)
+ "TOTAL" "Grand total")})
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]
+ (is (pending? model-id))
+ (run-jobs!)
+ (is (not (pending? model-id)))
+ (is (= "Grand total" (:display_name (column model-id "TOTAL")))))))))
+
+(deftest field-user-settings-test
+ (testing "renaming a field re-derives the MBQL cards reading its table"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Grand total"}]
+ (is (pending? card-id))
+ (run-jobs!)
+ (is (= "Grand total" (:display_name (column card-id "TOTAL"))))
+ (is (not (pending? card-id))))))))
+
+(deftest measure-edit-test
+ (testing "changing a measure's definition re-derives a card using it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Measure {measure-id :id} {:name "Measure"
+ :table_id (mt/id :orders)
+ :creator_id (mt/user->id :rasta)
+ :definition (-> (table-query :orders) (lib/aggregate (lib/count)))}
+ :model/Card {card-id :id} {:dataset_query (measure-query measure-id)}]
+ (run-jobs!)
+ (let [column-name #(:name (first (t2/select-one-fn :result_metadata :model/Card :id card-id)))]
+ (is (= "count" (column-name)))
+ (t2/update! :model/Measure measure-id
+ {:definition (-> (table-query :orders)
+ (lib/aggregate (lib/avg (lib.metadata/field (mt/metadata-provider)
+ (mt/id :orders :total)))))})
+ (run-jobs!)
+ (is (= "avg" (column-name))))))))
+
+(deftest measure-chain-edit-test
+ (testing "changing a measure's definition re-derives a card using a measure built on it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Measure {inner-id :id} {:name "Inner"
+ :table_id (mt/id :orders)
+ :creator_id (mt/user->id :rasta)
+ :definition (-> (table-query :orders) (lib/aggregate (lib/count)))}]
+ (mt/with-temp [:model/Measure {outer-id :id} {:name "Outer"
+ :table_id (mt/id :orders)
+ :creator_id (mt/user->id :rasta)
+ :definition (measure-query inner-id)}
+ :model/Card {card-id :id} {:dataset_query (measure-query outer-id)}]
+ (run-jobs!)
+ (let [base-type #(:base_type (first (t2/select-one-fn :result_metadata :model/Card :id card-id)))]
+ (is (isa? (base-type) :type/Integer))
+ (t2/update! :model/Measure inner-id
+ {:definition (-> (table-query :orders)
+ (lib/aggregate (lib/avg (lib.metadata/field (mt/metadata-provider)
+ (mt/id :orders :total)))))})
+ (run-jobs!)
+ (is (isa? (base-type) :type/Float))))))))
+
+(defn- rederive-during-inference!
+ "Re-derive the Card with `card-id` while `write!` runs in the middle of its inference, returning how many entities the
+ job took."
+ [card-id write!]
+ (reprocess! card-id)
+ (do-with-inference (fn [infer query card]
+ (write!)
+ (infer query card))
+ rederive!))
+
+(defn- store-run-metadata!
+ "Store `metadata` as the result metadata of the Card with `card-id`, as a run of the card does."
+ [card-id metadata]
+ (#'qp.db/set-card-result-metadata! card-id metadata))
+
+(defn- once!
+ "A function running `f` the first time it is called and doing nothing after."
+ [f]
+ (let [called (atom false)]
+ (fn [] (when (compare-and-set! called false true) (f)))))
+
+(deftest user-save-during-rederive-test
+ (testing "a user's save of a card's query during its re-derive is kept, and the card is re-derived again from it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]
+ (is (pos? (rederive-during-inference!
+ card-id
+ (once! #(t2/update! :model/Card card-id {:dataset_query (add-grand-total (table-query :orders))})))))
+ (is (some? (column card-id "Grand total")))
+ (is (pending? card-id))
+ (run-jobs!)
+ (is (= "Order total" (:display_name (column card-id "TOTAL"))))
+ (is (some? (column card-id "Grand total")))
+ (is (not (pending? card-id))))))))
+
+(deftest run-write-during-rederive-test
+ (testing "a run storing a card's metadata during its re-derive is kept, and the card is re-derived again from it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (let [run-metadata (-> (t2/select-one-fn :result_metadata :model/Card :id card-id)
+ (rename-column "TOTAL" "Run total")
+ (->> (mapv #(cond-> % (= "ID" (:name %)) (assoc :fingerprint {:global {:distinct-count 7}})))))]
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]
+ (is (pos? (rederive-during-inference! card-id (once! #(store-run-metadata! card-id run-metadata)))))
+ (run-jobs!)
+ (is (= "Order total" (:display_name (column card-id "TOTAL"))))
+ (is (= {:global {:distinct-count 7}} (:fingerprint (column card-id "ID"))))))))))
+
+(deftest card-on-card-temporal-unit-test
+ (testing "a card on a card takes the new temporal unit with the new display name"
+ (with-dependencies-feature
+ (let [mp (mt/metadata-provider)
+ by-unit #(-> (table-query :orders)
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket (lib.metadata/field mp (mt/id :orders :created_at)) %)))]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (by-unit :month)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (t2/update! :model/Card source-id {:dataset_query (by-unit :year)})
+ (run-jobs!)
+ (is (=? {:display_name "Created At: Year" :unit :year}
+ (column card-id "CREATED_AT"))))))))
+
+(deftest card-on-card-type-test
+ (testing "a card on a card takes the new type of a column whose type changed upstream"
+ (with-dependencies-feature
+ (let [total (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total))
+ with #(lib/expression (table-query :orders) "X" %)]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (with (lib/* total 2))}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (is (isa? (:base_type (column card-id "X")) :type/Number))
+ (t2/update! :model/Card source-id {:dataset_query (with (lib/concat total "!"))})
+ (run-jobs!)
+ (is (= :type/Text (:base_type (column card-id "X"))))
+ (testing "within one root type too, since no run measured the card's types"
+ (t2/update! :model/Card source-id {:dataset_query (with (lib/+ (lib.metadata/field (mt/metadata-provider)
+ (mt/id :orders :quantity))
+ 1))})
+ (run-jobs!)
+ (is (= :type/Integer (:base_type (column card-id "X"))))
+ (t2/update! :model/Card source-id {:dataset_query (with (lib/* total 1.5))})
+ (run-jobs!)
+ (is (= :type/Float (:base_type (column card-id "X"))))))))))
+
+(deftest alias-less-stored-metadata-test
+ (testing "stored columns without aliases are matched by name, so an alias equal to another column's name merges
+ nothing"
+ (let [stored [{:name "CREATED_AT" :id 36 :base_type "type/DateTime" :database_type "TIMESTAMP"
+ :fingerprint {:global {:distinct-count 1}}}
+ {:name "CREATED_AT_2" :id 14 :base_type "type/DateTime" :database_type "TIMESTAMP"
+ :fingerprint {:global {:distinct-count 2}}}]
+ inferred [{:name "CREATED_AT" :id 36 :base_type :type/DateTime
+ :lib/desired-column-alias "PRODUCTS__via__PRODUCT_ID__CREATED_AT"}
+ {:name "CREATED_AT_2" :id 14 :base_type :type/DateTime :lib/desired-column-alias "CREATED_AT"}]]
+ (is (=? [{:id 36 :fingerprint {:global {:distinct-count 1}}}
+ {:id 14 :fingerprint {:global {:distinct-count 2}}}]
+ (#'deps.result-metadata/patch-result-metadata stored inferred))))))
+
+(deftest kebab-case-stored-metadata-test
+ (testing "stored columns with kebab-case keys compare like snake_case ones"
+ (is (= [{:semantic_type "type/FK" :fk_target_field_id 29}]
+ (#'deps.result-metadata/comparable-metadata
+ (mapv #'deps.result-metadata/normalize-column [{:semantic-type :type/FK :fk-target-field-id 29}]))))))
+
+(deftest failure-waits-for-change-test
+ (testing "a card whose re-derive keeps failing is given up on, not re-derived again until it or an upstream changes
+ again, and then is"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (do-with-inference (fn [_infer _query _card] (throw (ex-info "Inference failed" {})))
+ run-jobs!)
+ (is (nil? (column card-id "Grand total")))
+ (is (not (pending? card-id)))
+ (run-jobs!)
+ (is (nil? (column card-id "Grand total")))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]
+ (run-jobs!)
+ (is (some? (column card-id "Grand total"))))))))
+
+(deftest result-metadata-job-before-dependency-job-test
+ (testing "a card whose dependencies the dependency job computes after its upstream changed is re-derived once it does"
+ (with-dependencies-feature
+ (mt/with-model-cleanup [:model/Card]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (let [card-id (t2/insert-returning-pk! :model/Card {:name "Dependent"
+ :display :table
+ :visualization_settings {}
+ :creator_id (mt/user->id :rasta)
+ :database_id (mt/id)
+ :dataset_query (card-query source-id)})]
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (mt/with-premium-features #{:dependencies}
+ (deps.result-metadata/update-result-metadata! 1000))
+ (run-jobs!)
+ (is (some? (column card-id "Grand total")))))))))
+
+(deftest new-edge-to-upstream-processed-before-save-test
+ (testing "a new dependency on a card processed before the dependent was saved has the card itself not processed again"
+ (with-dependencies-feature
+ (mt/with-model-cleanup [:model/Card]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card _ {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (is (not (pending? source-id)))
+ (Thread/sleep 5)
+ (t2/insert! :model/Card {:name "Dependent"
+ :display :table
+ :visualization_settings {}
+ :creator_id (mt/user->id :rasta)
+ :database_id (mt/id)
+ :dataset_query (card-query source-id)})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (not (pending? source-id))))))))
+
+(deftest new-dependent-of-upstream-processed-before-its-edges-test
+ (testing "a card whose upstream changed and was processed before the card's dependencies existed is re-derived when
+ they are computed, even when the card was saved again meanwhile"
+ (with-dependencies-feature
+ (mt/with-model-cleanup [:model/Card]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (let [card-id (t2/insert-returning-pk! :model/Card {:name "Dependent"
+ :display :table
+ :visualization_settings {}
+ :creator_id (mt/user->id :rasta)
+ :database_id (mt/id)
+ :dataset_query (card-query source-id)})]
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (deps.result-metadata/update-result-metadata! 1000)
+ (t2/update! :model/Card card-id {:name "Renamed"})
+ (run-jobs!)
+ (is (some? (column card-id "Grand total")))))))))
+
+(deftest job-reruns-while-it-takes-entities-test
+ (testing "the job runs again right away while it took entities, since a card it stored changed in turn"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card _ {:dataset_query (card-query source-id)}]
+ (deps.test/synchronously-run-dependency-job!)
+ (is (= 1 (#'task.result-metadata/run-batch!)))
+ (run-jobs!)
+ (is (nil? (#'task.result-metadata/run-batch!)))))))
+
+(deftest storing-card-triggers-jobs-test
+ (testing "the job stores a re-derived card through the Card model, so its hooks trigger the jobs, and keeps its
+ updated_at"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (deps.test/synchronously-run-dependency-job!)
+ (let [submitted (atom [])
+ before (t2/select-one-fn :updated_at :model/Card :id card-id)]
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (deps.test/synchronously-run-dependency-job!)
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]
+ (rederive!)
+ (rederive!))
+ (is (some? (column card-id "Grand total")))
+ (is (= before (t2/select-one-fn :updated_at :model/Card :id card-id)))
+ (is (seq @submitted)))))))
+
+(deftest sandboxed-creator-test
+ (testing "a card is re-derived as no user, so its creator's sandbox doesn't decide its columns, and the columns its
+ stored metadata lacks get no fingerprint"
+ (met/with-gtaps! {:gtaps {:venues {:query (let [mp (mt/metadata-provider)]
+ (lib/with-fields (table-query :venues)
+ [(lib.metadata/field mp (mt/id :venues :id))
+ (lib.metadata/field mp (mt/id :venues :name))
+ (lib.metadata/field mp (mt/id :venues :category_id))]))}}}
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)
+ :creator_id (mt/user->id :rasta)
+ :result_metadata []}]
+ (deps.test/synchronously-run-dependency-job!)
+ (reprocess! card-id)
+ (mt/with-premium-features #{:dependencies :sandboxes}
+ (deps.result-metadata/update-result-metadata! 1000))
+ (let [columns (t2/select-one-fn :result_metadata :model/Card :id card-id)]
+ (is (= ["ID" "NAME" "CATEGORY_ID" "LATITUDE" "LONGITUDE" "PRICE"] (mapv :name columns)))
+ (is (every? (comp nil? :fingerprint) columns))))))))
+
+(deftest legacy-stored-card-test
+ (testing "a card whose stored query is legacy MBQL with a filter is stored once and the job stops, though each read
+ gives the converted query new uuids"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (t2/update! (t2/table-name :model/Card) card-id
+ {:dataset_query (json/encode {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :venues)
+ :filter [:> [:field (mt/id :venues :price) nil] 1]}})
+ :result_metadata (json/encode [])})
+ (run-jobs!)
+ (is (seq (column card-id "PRICE")))
+ (is (not (pending? card-id)))
+ (is (nil? (#'task.result-metadata/run-batch!)))))))
+
+(deftest new-upstream-changed-before-its-edge-test
+ (testing "a card changed to read a card that changes before the dependency job stores the new edge is re-derived
+ from the changed one"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (t2/update! :model/Card card-id {:dataset_query (card-query source-id)})
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (run-jobs!)
+ (is (some? (column card-id "Grand total")))))))
+
+(deftest failing-store-test
+ (testing "a store that throws doesn't stop the other cards of the batch"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {failing-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {other-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! failing-id other-id)
+ (let [store (mt/original-fn #'dependencies.db/set-card-result-metadata!)]
+ (mt/with-dynamic-fn-redefs [dependencies.db/set-card-result-metadata!
+ (fn [card-id metadata]
+ (if (= failing-id card-id)
+ (throw (ex-info "Store failed" {}))
+ (store card-id metadata)))]
+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed"))
+ rederive!)))
+ (is (= "Changed" (:display_name (column other-id "NAME"))))))))
+
+(deftest own-edit-rederives-dependents-test
+ (testing "editing a card's query re-derives the cards depending on it, not the card, whose save inferred it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (let [inferred (atom #{})]
+ (do-with-inference (fn [infer query card]
+ (swap! inferred conj (:id card))
+ (infer query card))
+ run-jobs!)
+ (is (= #{card-id} @inferred))
+ (is (some? (column card-id "Grand total"))))))))
+
+(defn- native-run-metadata [sql]
+ (get-in (qp/process-query (lib/native-query (mt/metadata-provider) sql)) [:data :results_metadata :columns]))
+
+(deftest native-card-result-metadata-test
+ (testing "a write of a native card's run metadata re-derives its dependents"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {native-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT ID, NAME FROM VENUES")
+ :result_metadata (native-run-metadata "SELECT ID, NAME FROM VENUES")}
+ :model/Card {card-id :id} {:dataset_query (card-query native-id)}]
+ (run-jobs!)
+ (t2/update! :model/Card native-id {:dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT ID, NAME, PRICE FROM VENUES")})
+ (run-jobs!)
+ (is (nil? (column card-id "PRICE")))
+ (store-run-metadata! native-id (native-run-metadata "SELECT ID, NAME, PRICE FROM VENUES"))
+ (is (pending? native-id))
+ (run-jobs!)
+ (is (some? (column card-id "PRICE")))
+ (is (not (pending? card-id)))))))
+
+(deftest measured-keys-test
+ (testing "a re-derived column keeps what a run measured only while its root type and source type are unchanged, and only the stored fingerprint"
+ (let [stored {:name "X" :base_type "type/BigInteger" :database_type "BIGINT" :fingerprint {:type {:type/Number {}}}}
+ patch #(first (#'deps.result-metadata/patch-result-metadata [stored] [%]))]
+ (testing "a run's refinement within the root type and source type is kept"
+ (is (=? {:base_type "type/BigInteger" :fingerprint {:type {:type/Number {}}}}
+ (patch {:name "X" :base_type :type/Integer :database_type "BIGINT"}))))
+ (testing "a new root type takes the inferred type and keeps the stored fingerprint"
+ (is (= {:name "X" :base_type :type/Text :database_type "VARCHAR" :fingerprint {:type {:type/Number {}}}}
+ (patch {:name "X" :base_type :type/Text :database_type "VARCHAR" :fingerprint {:type {:type/Text {}}}}))))
+ (testing "a new source type within the root type takes the inferred type and keeps the stored fingerprint"
+ (is (= {:name "X" :base_type :type/Decimal :database_type "DECIMAL" :fingerprint {:type {:type/Number {}}}}
+ (patch {:name "X" :base_type :type/Decimal :database_type "DECIMAL"}))))
+ (testing "a column without a stored one gets no fingerprint"
+ (is (= {:name "Y" :base_type :type/Text}
+ (first (#'deps.result-metadata/patch-result-metadata
+ [stored] [{:name "Y" :base_type :type/Text :fingerprint {:type {:type/Text {}}}}]))))))))
+
+(deftest remapped-column-test
+ (testing "a card storing the metadata of a run with remapped columns is not rewritten, since inference adds them too"
+ (mt/with-column-remappings [orders.product_id products.title]
+ (let [query (table-query :orders)
+ run (get-in (qp/process-query query) [:data :results_metadata :columns])]
+ (is (some :remapped_from run))
+ (mt/with-temp [:model/Card card {:dataset_query query :result_metadata run}]
+ (is (nil? (#'deps.result-metadata/rederived-result-metadata card))))))))
+
+(deftest failed-dependencies-block-rederive-test
+ (testing "a card whose dependency computation keeps failing is not re-derived from its old edges until it changes"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]
+ (run-jobs!)
+ (let [calculate (mt/original-fn #'deps.calculation/calculate-deps)]
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps
+ (fn [entity-type entity]
+ (if (= card-id (:id entity))
+ (throw (ex-info "Computation failed" {}))
+ (calculate entity-type entity)))]
+ (t2/update! :model/Card card-id {:dataset_query (card-query source-id)
+ :description "edited"})
+ (dependencies.db/mark-dependency-statuses-stale! {:card #{card-id}})
+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})
+ (run-jobs!)))
+ (is (nil? (column card-id "Grand total")))
+ (is (pending? card-id))))))
+
+(deftest crash-leaves-card-pending-test
+ (testing "a job that dies between taking a card's status and clearing it leaves the card pending, and the next run
+ re-derives it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! card-id)
+ (is (thrown? Error (do-with-inference (fn [_infer _query _card] (throw (Error. "crash"))) rederive!)))
+ (is (pending? card-id))
+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed")) rederive!)
+ (is (= "Changed" (:display_name (column card-id "NAME"))))
+ (is (not (pending? card-id)))))))
+
+(deftest write-during-rederive-keeps-card-pending-test
+ (testing "a write counted on a card while it is re-derived keeps it pending after the job clears what it read"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! card-id)
+ (do-with-inference (fn [infer query card]
+ (dependencies.db/mark-result-metadata-statuses-stale! {:card #{card-id}})
+ (infer query card))
+ rederive!)
+ (is (pending? card-id))
+ (rederive!)
+ (is (not (pending? card-id)))))))
+
+(deftest store-compares-same-read-test
+ (testing "a card whose full read normalizes its stored metadata away is stored, since both compared reads are alike"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! card-id)
+ (let [card (mt/original-fn #'dependencies.db/card)]
+ (mt/with-dynamic-fn-redefs [dependencies.db/card (fn [id] (assoc (card id) :result_metadata nil))]
+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed"))
+ rederive!)))
+ (is (= "Changed" (:display_name (column card-id "NAME"))))
+ (is (not (pending? card-id)))))))
+
+(deftest conflict-fails-test
+ (testing "a card that keeps changing during its re-derive is left as the other writer stored it, and other cards proceed"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {orders-card-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {other-card-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (reprocess! orders-card-id other-card-id)
+ (do-with-inference (fn [infer query card]
+ (u/prog1 (rename-column (infer query card) "TOTAL" "Changed")
+ (when (= orders-card-id (:id card))
+ (store-run-metadata!
+ orders-card-id
+ (rename-column (:result_metadata card) "TOTAL" (str (random-uuid)))))))
+ rederive!)
+ (is (not= "Changed" (:display_name (column orders-card-id "TOTAL"))))
+ (is (= "Changed" (:display_name (column other-card-id "TOTAL"))))))))
+
+(deftest store-locks-only-dependents-test
+ (testing "storing a re-derived card locks only the statuses of its dependents, so a write of another status meanwhile
+ doesn't wait for it"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}
+ :model/Card {other-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! card-id)
+ (let [store (mt/original-fn #'dependencies.db/set-card-result-metadata!)
+ marked (atom nil)]
+ (mt/with-dynamic-fn-redefs [dependencies.db/set-card-result-metadata!
+ (fn [& args]
+ (reset! marked (deref (future (t2/update! :model/ResultMetadataStatus
+ :entity_type :card
+ :entity_id other-id
+ {:stale 0}))
+ 5000 :timed-out))
+ (apply store args))]
+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "TOTAL" "Changed"))
+ rederive!))
+ (is (= 1 @marked))
+ (is (= "Changed" (:display_name (column card-id "TOTAL")))))))))
+
+(deftest store-waits-for-save-in-flight-test
+ (testing "a save of the card holding its row lock while the job stores has the job re-derive it after the save"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]
+ (run-jobs!)
+ (reprocess! card-id)
+ (let [locked (promise)
+ save (future
+ (t2/with-transaction [_]
+ (t2/update! :model/Card card-id {:name "Saved"})
+ (deliver locked true)
+ (Thread/sleep 1000)))]
+ @locked
+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "TOTAL" "Changed"))
+ #(mt/with-premium-features #{:dependencies}
+ (deps.result-metadata/update-result-metadata! 1000)
+ @save
+ (deps.test/synchronously-run-dependency-job!)
+ (deps.result-metadata/update-result-metadata! 1000)))
+ (is (= "Saved" (t2/select-one-fn :name :model/Card :id card-id)))
+ (is (= "Changed" (:display_name (column card-id "TOTAL")))))))))
+
+(deftest failing-dependent-test
+ (testing "a card whose re-derive fails doesn't stop its siblings, and stays pending for the next run"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {first-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {failing-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {last-id :id} {:dataset_query (table-query :venues)}]
+ (run-jobs!)
+ (reprocess! first-id failing-id last-id)
+ (do-with-inference (fn [infer query card]
+ (if (= failing-id (:id card))
+ (throw (ex-info "Inference failed" {}))
+ (rename-column (infer query card) "NAME" "Changed")))
+ rederive!)
+ (is (= ["Changed" "Changed"] (mapv #(:display_name (column % "NAME")) [first-id last-id])))
+ (is (pending? failing-id))))))
+
+(deftest store-updates-parameters-using-dropped-columns-test
+ (testing "storing a re-derived card resets the parameters taking values from a column it no longer has"
+ (with-dependencies-feature
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}
+ :model/Card {card-id :id} {:dataset_query (table-query :venues)
+ :parameters [{:id "price"
+ :type :category
+ :name "Price"
+ :slug "price"
+ :values_source_type :card
+ :values_source_config {:card_id source-id
+ :value_field [:field (mt/id :venues :price) nil]}}]}]
+ (run-jobs!)
+ (reprocess! source-id)
+ (do-with-inference (fn [infer query card]
+ (cond->> (infer query card)
+ (= source-id (:id card)) (into [] (remove #(= "PRICE" (:name %))))))
+ rederive!)
+ (is (nil? (column source-id "PRICE")))
+ (is (=? [{:id "price" :values_source_type nil}]
+ (t2/select-one-fn :parameters :model/Card :id card-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj
index 956d3f0e5bc..deb1e3bbfb3 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj
@@ -2,157 +2,120 @@
(:require
[clojure.java.jdbc :as jdbc]
[clojure.test :refer [deftest is testing]]
- [java-time.api :as t]
- [metabase-enterprise.dependencies.db :as deps.db]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.hooks]
+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.test.data.one-off-dbs :as one-off-dbs]
- [metabase.util :as u]
[toucan2.core :as t2]))
+(defn- pending?
+ "Whether the Card with `card-id` is to be re-analyzed and re-derived."
+ [card-id]
+ (and (deps.test/analysis-finding-pending? :card card-id)
+ (deps.test/result-metadata-pending? card-id)))
+
+(defn- record-processed!
+ "Record that the analysis and result metadata jobs processed the Card with `card-id`."
+ [card-id]
+ (deps.test/record-analysis-finding-processed! :card card-id)
+ (deps.test/record-result-metadata-processed! card-id))
+
(defn- with-syncable-db! [thunk]
- (mt/with-premium-features #{:dependencies}
- (one-off-dbs/with-blank-db
- ;; Step 1: Create a table with two columns
- (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"
- "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]
- (jdbc/execute! one-off-dbs/*conn* [statement]))
- ;; Sync the database to pick up the new table
- (sync/sync-database! (mt/db))
- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")
- filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")
- ;; Step 2: Create a card with a filter on filter_col
- mp (lib-be/application-database-metadata-provider (mt/id))
- table (lib.metadata/table mp table-id)
- filter-field (lib.metadata/field mp filter-field-id)
- query (-> (lib/query mp table)
- (lib/filter (lib/= filter-field "value1")))]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query
- :database_id (mt/id)}]
- ;; Create the dependency manually
- (t2/insert! :model/Dependency {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id table-id})
- ;; Step 3: Run analysis - should succeed with no errors
- (let [card (t2/select-one :model/Card card-id)]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:dependencies}
+ (one-off-dbs/with-blank-db
+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"
+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]
+ (jdbc/execute! one-off-dbs/*conn* [statement]))
+ (sync/sync-database! (mt/db))
+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")
+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")
+ mp (lib-be/application-database-metadata-provider (mt/id))
+ query (-> (lib/query mp (lib.metadata/table mp table-id))
+ (lib/filter (lib/= (lib.metadata/field mp filter-field-id) "value1")))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query
+ :database_id (mt/id)}]
+ (t2/insert! :model/Dependency {:from_entity_type :card
+ :from_entity_id card-id
+ :to_entity_type :table
+ :to_entity_id table-id})
;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see
;; old values after the sync!
(lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! card)))
- (testing "Initial analysis succeeds"
- (is (true?
- (t2/select-one-fn :result :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)))
- (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))
- ;; Backdate the analysis timestamp and table/field updated_at so
- ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)
- ;; after a new sync makes an edit.
- ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and
- ;; the field update from sync would have identical timestamps.
- (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]
- (t2/update! :model/AnalysisFinding
- {:analyzed_entity_type :card :analyzed_entity_id card-id}
- {:analyzed_at old-time})
- (t2/update! :model/Field :table_id table-id
- {:updated_at old-time})
- (t2/update! :model/Table :id table-id
- {:updated_at old-time}))
- ;; Setup complete: Now call the thunk for the actual test run.
- (thunk {:db-id (:id (mt/db))
- :card-id card-id
- :table-id table-id
- :filter-field-id filter-field-id}))))))
+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Card card-id)))
+ (testing "Initial analysis succeeds"
+ (is (true? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_type :card :entity_id card-id)))
+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))
+ (record-processed! card-id)
+ (thunk {:card-id card-id
+ :table-id table-id
+ :filter-field-id filter-field-id})))))))
-;; Integration test that a real DB sync triggers re-analysis of an updated table.
(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test
- (testing "When sync detects a removed column, re-analyzing the card shows errors"
+ (testing "when sync deactivates a column, the cards on its table are re-analyzed and show the missing column"
(with-syncable-db!
(fn [{:keys [card-id filter-field-id]}]
- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- ;; Step 4: Remove the column used in the filter
- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
- ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis
- ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.
- (sync/sync-database! (mt/db))
- ;; Verify the field is now inactive
- (testing "Field is marked inactive after sync"
- (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))
- ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.
- ;; Due to race conditions, the entity may be either stale (job hasn't run yet)
- ;; or already reanalyzed (job ran). Check for either valid outcome.
- (let [finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (is (some? finding) "Analysis finding should exist")
- (if (:stale finding)
- ;; Job hasn't run yet - sync marked it stale, that's the expected behavior
- (testing "Entity is marked stale for re-analysis"
- (is (true? (:stale finding))))
- ;; Job already ran - verify the reanalysis detected the missing column
- (testing "Re-analysis detects the missing column"
- (is (false? (:result finding)) "Analysis should fail")
- (is (not= initial-analysis-time (:analyzed_at finding))
- "Analysis should have been re-run")
- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
- (is (seq errors) "Should have analysis errors")
- (is (some #(= :missing-column (:error_type %)) errors)
- "Should have a missing-column error"))))))))))
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
+ (sync/sync-database! (mt/db))
+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id)))
+ (is (pending? card-id))
+ (deps.test/synchronously-run-dependency-job!)
+ (deps.test/run-analysis-finding-job!)
+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding
+ :entity_type :card :entity_id card-id)))
+ (is (some #(= :missing-column (:error_type %))
+ (deps.analysis-finding-error/errors-for-entity :card card-id)))
+ (is (not (deps.test/analysis-finding-pending? :card card-id)))))))
(deftest ^:synchronized scheduled-metadata-sync-triggers-reanalysis-test
- (testing "The metadata sync the scheduled sync task runs marks the dependents of a changed table for re-analysis"
+ (testing "the metadata sync the scheduled sync task runs has the cards on a table it changes processed"
(with-syncable-db!
(fn [{:keys [card-id]}]
- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
- (sync/sync-db-metadata! (mt/db))
- (let [finding (t2/select-one :model/AnalysisFinding :analyzed_entity_type :card :analyzed_entity_id card-id)]
- (is (or (:stale finding)
- (not= initial-analysis-time (:analyzed_at finding))))))))))
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
+ (sync/sync-db-metadata! (mt/db))
+ (is (pending? card-id))))))
-;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all
-;; cards which depend on that table.
(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test
- (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"
+ (testing "a sync that changes nothing about a table or its fields has nothing processed"
+ (with-syncable-db!
+ (fn [{:keys [card-id filter-field-id table-id]}]
+ (let [table-before (into {} (t2/select-one :model/Table :id table-id))
+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]
+ (sync/sync-database! (mt/db))
+ (is (= table-before (into {} (t2/select-one :model/Table :id table-id))))
+ (is (= filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))))
+ (is (not (pending? card-id)))
+ (is (not (deps.test/analysis-finding-pending? :card card-id))))))))
+
+(deftest ^:synchronized table-sync-makes-cards-on-changed-table-pending-test
+ (testing "a sync of one table, like after an upload or a transform run, has the cards on it processed only when it
+ changes it"
+ (with-syncable-db!
+ (fn [{:keys [card-id table-id]}]
+ (sync/sync-table! (t2/select-one :model/Table :id table-id))
+ (is (not (pending? card-id)))
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" ADD COLUMN \"new_col\" VARCHAR;"])
+ (sync/sync-table! (t2/select-one :model/Table :id table-id))
+ (is (pending? card-id))))))
+
+(deftest ^:synchronized sync-discovered-fk-makes-cards-on-its-table-pending-test
+ (testing "a foreign key a sync discovers has the cards on the table of its field processed"
(with-syncable-db!
- (fn [{:keys [card-id db-id filter-field-id table-id]}]
- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)
- db-deps-checked (atom [])
- original-deps-check (mt/original-fn #'deps.db/table-ids-with-outdated-findings)
- table-before (into {} (t2/select-one :model/Table :id table-id))
- filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]
- ;; Re-sync the database, having made no changes to it.
- (mt/with-dynamic-fn-redefs [deps.db/table-ids-with-outdated-findings
- (fn [db-id]
- (u/prog1 (original-deps-check db-id)
- (swap! db-deps-checked conj [db-id <>])))]
- (sync/sync-database! (mt/db)))
- (testing "sync doesn't update tables or fields that haven't changed"
- (let [table-after (into {} (t2/select-one :model/Table :id table-id))
- filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]
- (is (= table-before table-after))
- (is (= filter-field-before filter-field-after))))
- (testing "the DB is were checked for tables that need deps updates, but none were found"
- (is (=? [[db-id empty?]]
- @db-deps-checked)))
- (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"
- (let [finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (is (some? finding) "Analysis finding should exist")
- (is (true? (:result finding)) "Analysis should be a success")
- (is (= initial-analysis-time (:analyzed_at finding))
- "Analysis should not have been re-run"))))))))
+ (fn [{:keys [card-id]}]
+ (jdbc/execute! one-off-dbs/*conn* ["CREATE TABLE \"ref_table\" (\"id\" INTEGER PRIMARY KEY, \"test_id\" INTEGER);"])
+ (sync/sync-database! (mt/db))
+ (let [ref-table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "ref_table")
+ ref-field-id (t2/select-one-fn :id :model/Field :table_id ref-table-id :name "test_id")]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id ref-table-id})
+ (record-processed! card-id)
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"ref_table\" ADD FOREIGN KEY (\"test_id\") REFERENCES \"test_table\"(\"id\");"])
+ (sync/sync-database! (mt/db))
+ (is (some? (t2/select-one-fn :fk_target_field_id :model/Field :id ref-field-id)))
+ (is (pending? card-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.clj
new file mode 100644
index 00000000000..a86b93136ab
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.clj
@@ -0,0 +1,52 @@
+(ns metabase-enterprise.dependencies.task.analysis-finding-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [environ.core :as env]
+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]
+ [metabase-enterprise.dependencies.task.util :as task.util]
+ [metabase.dependencies.hooks :as dependencies.hooks]
+ [metabase.task.core :as task]
+ [metabase.test :as mt])
+ (:import
+ (org.quartz JobDetail)))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:synchronized batch-size-zero-disables-job-test
+ (testing "a batch size that isn't positive keeps the job from being scheduled by writes or at startup"
+ (let [scheduled (atom [])
+ scheduled-by (fn [thunk]
+ (reset! scheduled [])
+ (thunk)
+ (count @scheduled))]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
+ (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "0")]
+ (is (zero? (scheduled-by task.analysis-finding/trigger-analysis-finding-job!)))
+ (is (zero? (scheduled-by #(task/init! ::task.analysis-finding/UpdateAnalysisFindings)))))
+ (testing "a positive batch size schedules from the event trigger"
+ (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "5")]
+ (is (= 1 (scheduled-by task.analysis-finding/trigger-analysis-finding-job!)))))))))
+
+(deftest ^:synchronized run-stops-without-feature-test
+ (testing "a run without the dependencies feature schedules no next run, since the next write or token update triggers one"
+ (let [scheduled (atom [])]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
+ (mt/with-premium-features #{}
+ (task.util/run-and-reschedule! nil @#'task.analysis-finding/job))
+ (is (empty? @scheduled))))))
+
+(def ^:private job-key "metabase.dependencies.task.entity-check.job")
+
+(defn- scheduled-job-names
+ "Job names from captured [[task/schedule-task!]] calls, since a token update schedules every dependency job."
+ [calls]
+ (into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))
+
+(deftest ^:synchronized token-update-restarts-job-test
+ (testing "Setting a premium token must restart the analysis finding job, or enabling the feature on an idle instance
+ leaves the periodic checker dead until the next process restart."
+ (let [scheduled (atom [])]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
+ (mt/with-premium-features #{:dependencies}
+ (dependencies.hooks/after-premium-token-update)
+ (is (contains? (scheduled-job-names @scheduled) job-key)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
deleted file mode 100644
index 8f2ec778799..00000000000
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ /dev/null
@@ -1,645 +0,0 @@
-(ns metabase-enterprise.dependencies.task.backfill-test
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.backfill-test]}}}}}}
- (:require
- [clojure.string :as str]
- [clojure.test :refer :all]
- [environ.core :as env]
- [java-time.api :as t]
- [metabase-enterprise.dependencies.calculation :as deps.calculation]
- [metabase-enterprise.dependencies.models.dependency :as dependencies.model]
- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
- [metabase-enterprise.dependencies.test-util :as deps.test]
- [metabase.dependencies.hooks :as dependencies.hooks]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.premium-features.core :as premium-features]
- [metabase.task.core :as task]
- [metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]
- [toucan2.core :as t2])
- (:import
- (org.quartz JobKey)))
-
-(set! *warn-on-reflection* true)
-
-(use-fixtures :once (fixtures/initialize :db))
-
-(defn- backfill-all-existing-entities!
- []
- (deps.test/synchronously-run-backfill!))
-
-(defn- backfill-dependencies-single-trigger!
- []
- (mt/with-premium-features #{:dependencies}
- (#'dependencies.backfill/backfill-dependencies!)))
-
-(defn- mark-stale!
- "Mark an entity as stale in dependency_status for testing."
- [entity-type entity-id]
- (deps.dependency-status/mark-stale! {entity-type #{entity-id}}))
-
-(defn- assert-processed
- "Assert that an entity has been processed (not stale, current version)."
- [entity-type entity-id]
- (is (t2/exists? :model/DependencyStatus
- :entity_type entity-type
- :entity_id entity-id
- :stale false
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version)
- (str "Expected " (name entity-type) " " entity-id " to be processed")))
-
-(deftest ^:synchronized backfill-dependency-analysis-test
- (testing "Test that the backfill job correctly processes stale entities"
- (backfill-all-existing-entities!)
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
- (let [query (mt/mbql-query orders)]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
- :model/Card {card2-id :id} {:dataset_query query}
- :model/Card {card3-id :id} {:dataset_query query}]
- ;; Mark all three cards as stale
- (mark-stale! :card card1-id)
- (mark-stale! :card card2-id)
- (mark-stale! :card card3-id)
- (let [stale-count (fn []
- (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
- :stale true))]
- (is (= 3 (stale-count)))
- ;; first run, should process 2 cards
- (is (true? (backfill-dependencies-single-trigger!)))
- (is (= 1 (stale-count)))
- ;; second run, should process the last card
- (is (false? (backfill-dependencies-single-trigger!)))
- (is (= 0 (stale-count)))
- ;; third run, should not process anything
- (is (false? (backfill-dependencies-single-trigger!)))
- (is (= 0 (stale-count))))))))))
-
-(deftest ^:synchronized backfill-transform-test
- (testing "Test that transforms are correctly backfilled"
- (backfill-all-existing-entities!)
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
- :model/Transform {transform-id :id} {:name "Test Transform"
- :source {:type "query"
- :query (mt/mbql-query nil
- {:source-table (str "card__" card-id)})}
- :target {:type "table" :name (mt/random-name)}}]
- ;; Mark both as stale
- (mark-stale! :card card-id)
- (mark-stale! :transform transform-id)
- (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))
- (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (assert-processed :transform transform-id)
- (is (t2/exists? :model/Dependency
- :from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id (mt/id :orders)))
- (is (t2/exists? :model/Dependency
- :from_entity_type :transform :from_entity_id transform-id
- :to_entity_type :card :to_entity_id card-id)))))))
-
-(deftest ^:synchronized backfill-snippet-test
- (testing "Test that cards with snippets are correctly backfilled"
- (backfill-all-existing-entities!)
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"
- :content "SELECT 1"}
- :model/Card {card-id :id} {:dataset_query {:database (mt/id)
- :type :native
- :native {:query "SELECT * FROM {{my_snippet}}"
- :template-tags
- {:my_snippet
- {:id (str (random-uuid))
- :name "my_snippet"
- :display-name "my_snippet"
- :type :snippet
- :snippet-name "my_snippet"
- :snippet-id snippet-id}}}}}]
- ;; Mark both as stale
- (mark-stale! :card card-id)
- (mark-stale! :snippet snippet-id)
- (is (false? (t2/exists? :model/Dependency
- :from_entity_type :card :from_entity_id card-id
- :to_entity_type :snippet :to_entity_id snippet-id)))
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (assert-processed :snippet snippet-id)
- (is (t2/exists? :model/Dependency
- :from_entity_type :card :from_entity_id card-id
- :to_entity_type :snippet :to_entity_id snippet-id)))))))
-
-(deftest ^:synchronized backfill-snippet-in-snippet-test
- (testing "A snippet referencing another snippet produces a snippet->snippet dependency"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner_snip" :content "1 = 1"}
- :model/NativeQuerySnippet {outer-id :id} {:name "outer_snip" :content "SELECT * WHERE {{snippet: inner_snip}}"}]
- (mark-stale! :snippet inner-id)
- (mark-stale! :snippet outer-id)
- (is (false? (t2/exists? :model/Dependency
- :from_entity_type :snippet :from_entity_id outer-id
- :to_entity_type :snippet :to_entity_id inner-id)))
- (backfill-dependencies-single-trigger!)
- (assert-processed :snippet inner-id)
- (assert-processed :snippet outer-id)
- (is (t2/exists? :model/Dependency
- :from_entity_type :snippet :from_entity_id outer-id
- :to_entity_type :snippet :to_entity_id inner-id))))))
-
-(deftest ^:synchronized backfill-snippet-referencing-card-test
- (testing "A snippet referencing a card produces a snippet->card dependency"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
- :model/NativeQuerySnippet {snippet-id :id} {:name "card_ref_snip"
- :content (format "SELECT * FROM {{#%d}}" card-id)}]
- (mark-stale! :card card-id)
- (mark-stale! :snippet snippet-id)
- (is (false? (t2/exists? :model/Dependency
- :from_entity_type :snippet :from_entity_id snippet-id
- :to_entity_type :card :to_entity_id card-id)))
- (backfill-dependencies-single-trigger!)
- (assert-processed :snippet snippet-id)
- (is (t2/exists? :model/Dependency
- :from_entity_type :snippet :from_entity_id snippet-id
- :to_entity_type :card :to_entity_id card-id))))))
-
-(deftest ^:synchronized backfill-idempotency-test
- (testing "Running the backfill multiple times should be idempotent"
- (backfill-all-existing-entities!)
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id (mt/id :orders))))
- ;; First run
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- ;; Second run - should not change anything
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id (mt/id :orders))))))))
-
-(defn- wait-for-condition
- [predicate timeout-ms]
- (let [limit (+ (System/currentTimeMillis) timeout-ms)]
- (loop []
- (Thread/sleep 100)
- (or (predicate)
- (when (< (System/currentTimeMillis) limit)
- (recur))))))
-
-(deftest ^:synchronized backfill-scheduling-test
- (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"
- (backfill-all-existing-entities!)
- (mt/with-temp-scheduler!
- (with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-batch-size "1"
- :mb-dependency-backfill-delay-minutes "0"
- :mb-dependency-backfill-variance-minutes "0")]
- (let [card-data {:dataset_query (mt/mbql-query orders)}]
- (mt/test-helpers-set-global-values!
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card1-id :id} card-data
- :model/Card {card2-id :id} card-data]
- (mark-stale! :card card1-id)
- (mark-stale! :card card2-id)
- (let [processed? (fn []
- (= 2 (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id]]
- :stale false
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version)))]
- (is (not (processed?)))
- (mt/with-premium-features #{:dependencies}
- (task/init! ::dependencies.backfill/DependencyBackfill)
- (wait-for-condition processed? 2500)
- (is (processed?))))))))))))
-
-(deftest ^:synchronized backfill-no-trigger-pile-up-test
- (testing "Scheduling new runs while the backfill job is executing keeps the trigger count ≤ 1"
- (let [job-started (promise)
- can-finish (promise)]
- ;; the job body runs on a Quartz worker thread, which does not inherit *local-redefs* —
- ;; only a root swap is visible there
- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [dependencies.backfill/backfill-dependencies!
- (fn []
- (deliver job-started true)
- (deref can-finish 1500 :timeout)
- ;; non-nil → the job will hit the in-job self-reschedule path on exit
- true)]
- (mt/with-temp-scheduler!
- ;; Start the job immediately.
- (#'dependencies.backfill/schedule-run! (task/scheduler) 0)
- (is (true? (deref job-started 1000 :timeout))
- "BackfillDependencies job did not start within 1s")
- (let [scheduler (task/scheduler)
- job-key (JobKey. "metabase.task.dependency-backfill.job")
- trigger-count #(count (.getTriggersOfJob scheduler job-key))]
- ;; Spam 50 event-driven schedule attempts while the job is in-flight.
- (dotimes [_ 50]
- (dependencies.backfill/trigger-backfill-job!))
- (is (<= (trigger-count) 1)
- (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))
- ;; Release the job; on exit it self-reschedules via the in-job path.
- (deliver can-finish true)
- (Thread/sleep 200)
- (is (<= (trigger-count) 1)
- (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))
-
-(deftest ^:synchronized backfill-error-logging-test
- (testing "When calculate-deps throws, the error is logged and the entity remains stale"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-delay-minutes "0"
- :mb-dependency-backfill-variance-minutes "0")
- deps.calculation/calculate-deps
- (fn [_ _] (throw (ex-info "Simulated error" {})))]
- (backfill-dependencies-single-trigger!))
- ;; Entity should still be stale — not processed, not lost
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; No dependencies should have been created
- (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))
- ;; Failure recorded in table
- (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus
- :entity_type :card :entity_id card-id)))))))
-
-(deftest ^:synchronized backfill-partial-batch-failure-test
- (testing "A failure on one entity doesn't prevent other entities from being processed"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}
- :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]
- (mark-stale! :card good-card-id)
- (mark-stale! :card bad-card-id)
- (let [original-calculate-deps deps.calculation/calculate-deps]
- (with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-delay-minutes "0"
- :mb-dependency-backfill-variance-minutes "0")
- deps.calculation/calculate-deps
- (fn [entity-type entity]
- (if (= (:id entity) bad-card-id)
- (throw (ex-info "Simulated error" {}))
- (original-calculate-deps entity-type entity)))]
- (backfill-dependencies-single-trigger!)))
- ;; Good card should be processed successfully
- (assert-processed :card good-card-id)
- (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))
- ;; Bad card should remain stale with failure recorded
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id bad-card-id :stale true))
- (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus
- :entity_type :card :entity_id bad-card-id)))))))
-
-(deftest ^:synchronized backfill-terminal-failure-test
- (testing "Entities should be marked as terminally broken after MAX_RETRIES failures"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- (let [compute-attempts (volatile! 0)
- failures (inc @#'dependencies.backfill/max-retries)]
- (with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-delay-minutes "0"
- :mb-dependency-backfill-variance-minutes "0")
- ;; Make compute-deps-for-entity! throw
- deps.calculation/calculate-deps
- (fn [_ _]
- (vswap! compute-attempts inc)
- (throw (ex-info "Simulated computation error" {:card-id card-id})))]
- ;; fail MAX_RETRIES + 1 times
- (while (< @compute-attempts failures)
- (backfill-dependencies-single-trigger!))))
- ;; verify card is still stale (not processed)
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; verify subsequent runs don't process it
- (backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))))))
-
-(deftest ^:synchronized backfill-delayed-retry-test
- (testing "Failed entities should be retried after their delay period expires"
- (backfill-all-existing-entities!)
- (let [compute-attempts (volatile! 0)]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-delay-minutes "1")
- deps.calculation/calculate-deps
- (fn [_ _entity]
- (if (zero? @compute-attempts)
- (do
- (vswap! compute-attempts inc)
- (throw (ex-info "Simulated computation error" {:card-id card-id})))
- ;; Return valid deps on subsequent attempts
- {:table #{(mt/id :orders)}}))]
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; first failure - should be put into retry state
- (while (zero? @compute-attempts)
- (backfill-dependencies-single-trigger!))
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; advance time by less than retry delay - should NOT be processed
- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))
- (backfill-dependencies-single-trigger!))
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; advance time by more than retry delay - should be processed
- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))
- (backfill-dependencies-single-trigger!))
- (assert-processed :card card-id)))))))
-
-(deftest backfill-card-does-not-cause-revision-test
- (testing "backfilling a card does not create a new revision or audit log entry"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{:dependencies :audit-app}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)
- deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
- (is (= 0 revision-count-before))
- (is (= 0 deps-before))
- (backfill-dependencies-single-trigger!)
- (let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)
- deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
- (is (= 0 revision-count-after))
- (assert-processed :card card-id)
- (is (= 1 deps-after))))))))
-
-(deftest ^:synchronized backfill-dependencies-on-token-update-test
- (testing "Token update triggers the backfill job (entities remain stale until job runs)"
- (backfill-all-existing-entities!)
- (let [query (mt/mbql-query orders)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
- :model/Card {card2-id :id} {:dataset_query query}
- :model/Card {card3-id :id} {:dataset_query query}]
- ;; Mark all as stale
- (mark-stale! :card card1-id)
- (mark-stale! :card card2-id)
- (mark-stale! :card card3-id)
- ;; token update triggers the async job, not synchronous processing
- (dependencies.hooks/after-premium-token-update)
- ;; entities are still stale (job hasn't run yet in this test)
- (is (= 3 (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
- :stale true)))
- ;; manually run the backfill to verify it processes them
- (backfill-all-existing-entities!)
- (is (= 3 (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
- :stale false
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))
-
-(deftest ^:synchronized backfill-version-outdated-test
- (testing "Entities with outdated version in dependency_status get reprocessed"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- ;; Create a dependency_status entry with an old version (not stale, but outdated)
- (t2/insert! :model/DependencyStatus {:entity_type :card
- :entity_id card-id
- :dependency_analysis_version 0
- :stale false})
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id (mt/id :orders)))))))
-
-(deftest ^:synchronized backfill-no-status-row-test
- (testing "Entities with no dependency_status row yet get picked up and processed by the backfill"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- ;; Precondition: the temp card was inserted directly (no :event/card-create), so it has no
- ;; status row at all. This is the case the left-join in instances-for-dependency-calculation
- ;; must catch — guard it so the test can't silently pass without exercising that branch.
- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))
- "Expected the temp card to start with no dependency_status row")
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id (mt/id :orders)))))))
-
-(deftest ^:synchronized has-stale-or-outdated?-counts-no-status-row-test
- (testing "has-stale-or-outdated? (backing /backfill-status) stays consistent with what the backfill processes"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- ;; Clean slate: nothing left to process.
- (is (false? (deps.dependency-status/has-stale-or-outdated?))
- "Expected no pending work after backfilling all existing entities")
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- ;; A card with no status row is pending work, even though no DependencyStatus row exists.
- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id)))
- (is (true? (deps.dependency-status/has-stale-or-outdated?))
- "Expected pending work: a card with no status row still needs calculation")
- (backfill-dependencies-single-trigger!)
- (assert-processed :card card-id)
- (is (false? (deps.dependency-status/has-stale-or-outdated?))
- "Expected no pending work once the card has been processed")))))
-
-(deftest ^:synchronized batch-size-zero-suppresses-event-triggers-but-stays-warm-test
- (testing "A non-positive batch size stops the job doing work, but leaves it on its slow periodic schedule so it
- resumes if the setting ever becomes positive. What it must not do is keep firing the 1-second event-driven
- trigger, which is what made a disabled job wake roughly once a second on a busy instance -- entity changes
- fire it, and it never consulted the batch size."
- (let [scheduled (atom [])
- scheduled-by (fn [thunk]
- (reset! scheduled [])
- (thunk)
- (count @scheduled))]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]
- (when (some #(str/includes? (str %) "dependency-backfill") args)
- (swap! scheduled conj args))
- nil)]
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")]
- (testing "the 1-second event-driven trigger is suppressed"
- (is (zero? (scheduled-by dependencies.backfill/trigger-backfill-job!))))
- (testing "but task/init! still puts the job on its periodic schedule"
- (is (= 1 (scheduled-by #(task/init! ::dependencies.backfill/DependencyBackfill)))))
- (testing "and a run keeps the chain going rather than stopping"
- (mt/with-premium-features #{:dependencies}
- (is (= 1 (scheduled-by #(#'dependencies.backfill/run-and-reschedule! nil)))))))
- (testing "a positive batch size schedules from the event trigger too"
- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "5")]
- (is (= 1 (scheduled-by dependencies.backfill/trigger-backfill-job!)))))))))
-
-(deftest ^:synchronized backfill-batch-shares-one-metadata-provider-test
- (testing "GHY-4251: every entity in a batch must share one MetadataProvider per database. Reading an entity attaches
- a provider to its query, so without a batch-scoped cache each entity gets a private one; native dep
- calculation then fills each with that database's table metadata and the batch retains one copy per entity,
- which is what exhausted the heap on instances with large warehouses."
- (backfill-all-existing-entities!)
- (let [providers (atom [])
- calculate-deps (mt/original-fn #'deps.calculation/calculate-deps)
- native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}
- :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]
- (mark-stale! :card card1-id)
- (mark-stale! :card card2-id)
- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps
- (fn [entity-type entity]
- (when-let [mp (:lib/metadata (:dataset_query entity))]
- (swap! providers conj mp))
- (calculate-deps entity-type entity))]
- (backfill-dependencies-single-trigger!))
- (testing "both cards were analyzed, so the assertion below is meaningful"
- (is (<= 2 (count @providers))))
- (testing "and they were analyzed through the same provider instance"
- ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be
- ;; satisfied by two separate caches and would not detect the regression.
- (is (= 1 (count (into #{} (map #(System/identityHashCode %)) @providers))))))))))
-
-(deftest ^:synchronized backfill-records-failure-for-fatal-error-test
- (testing "GHY-4251: a fatal Error must record the entity's failure before propagating. Only Exception was caught,
- so an OutOfMemoryError escaped without recording anything and the next run selected the identical batch --
- a crash loop that never made progress. Recording first means the entity backs off and eventually goes
- terminal, so the batch drains even if one entity always kills the process."
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (Error. "simulated fatal error")))]
- (testing "the Error propagates instead of being swallowed, failing the job"
- (is (thrown-with-msg? Error #"simulated fatal error"
- (backfill-dependencies-single-trigger!)))))
- (testing "and the failure was recorded first, so the next run backs off instead of replaying the batch"
- (let [{:keys [fail_count next_retry_at terminal]}
- (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)]
- (is (= 1 fail_count))
- (is (some? next_retry_at))
- (is (false? terminal))))))))
-
-(deftest ^:synchronized pending-retries-licence-states-test
- (testing "A pending retry keeps the backfill job scheduled, but only when the licence permits acting on it. A retry
- marker is cleared by processing the entity -- on success via upsert-status!, or by going terminal past
- max-retries -- and both are gated on the feature, so without it the job could never resolve the condition
- keeping it awake and rescheduled hourly forever.
-
- The licence is resolved with canonically-has-feature? rather than has-feature?, which collapses 'no licence'
- and 'could not check' into the same false. Both inputs to the job's reschedule decision consult the licence,
- so a transient token failure would take the job down permanently -- nothing re-fires when the check
- recovers, because the token never changed."
- (backfill-all-existing-entities!)
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (mark-stale! :card card-id)
- (backfill-dependencies-single-trigger!)
- (deps.dependency-status/record-failure! :card card-id 5 60)
- (is (t2/exists? :model/DependencyStatus
- :entity_type :card :entity_id card-id
- :terminal false :next_retry_at [:not= nil])
- "test setup: the card must be in retry backoff")
- (let [actionable? (fn [licence]
- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly licence)]
- (#'dependencies.backfill/has-pending-retries?)))]
- (testing "licensed: the pending retry keeps the job scheduled"
- (is (true? (actionable? true))))
- (testing "indeterminate: treated as actionable, so a blip cannot end the chain"
- (is (true? (actionable? nil))))
- (testing "definitively unlicensed: not actionable"
- (is (false? (actionable? false))))))))
-
-(deftest ^:synchronized backfill-records-failure-for-never-processed-entity-test
- (testing "An entity with no dependency_status row yet must still get a failure recorded when it fails. Such entities
- are explicitly selected for processing (instances-for-dependency-calculation matches a null status row), so
- on a first backfill every entity is in this state -- exactly the population most likely to fail on a large
- instance. record-failure! only updated an existing row, so nothing was recorded and the next run selected
- the identical batch: the crash loop, unbroken for the entities most likely to cause it."
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))
- "test setup: the card must have no status row, which is what makes it eligible")
- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (ex-info "boom" {})))]
- (backfill-dependencies-single-trigger!))
- (testing "the failure is recorded, so the entity backs off instead of being reselected unchanged"
- (let [{:keys [fail_count next_retry_at]}
- (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)]
- (is (= 1 fail_count))
- (is (some? next_retry_at))))))))
-
-(deftest ^:synchronized fatal-error-still-reschedules-test
- (testing "A failure must not leave the job unscheduled. The job is one-shot self-rescheduling with no cron
- backstop, so an Error propagating out of the batch skipped schedule-run! entirely and the chain ended
- until a content-change event or a restart. A real OutOfMemoryError usually takes the process with it and
- task/init! recovers on boot, but survivable Errors -- StackOverflowError from deeply nested SQL, an
- AssertionError, a LinkageError -- leave a live process with a dead job."
- (let [scheduled (atom [])]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)
- dependencies.backfill/backfill-dependencies!
- (fn [& _] (throw (Error. "simulated fatal error")))]
- (testing "the Error still propagates"
- (is (thrown-with-msg? Error #"simulated fatal error"
- (#'dependencies.backfill/run-and-reschedule! nil))))
- (testing "and the next run was scheduled anyway"
- (is (= 1 (count @scheduled))))))))
-
-(defn- table-dependency-ids
- "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."
- [entity-type entity-id]
- (t2/select-fn-set :to_entity_id :model/Dependency
- :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))
-
-(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test
- (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,
- recomputes the cards whose SQL expands it"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}
- :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}
- :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
- "{{snippet: birds outer}}")}]
- (backfill-all-existing-entities!)
- (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))
- (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})
- (backfill-all-existing-entities!)
- (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))
-
-(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test
- (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
- :model/Card {card-id :id} {:dataset_query
- (lib/native-query
- mp
- (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"
- orders-id))}]
- (backfill-all-existing-entities!)
- (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))
- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
- :to_entity_type :card :to_entity_id orders-id)))))))
-
-(deftest ^:synchronized backfill-records-table-sync-finds-later-test
- (testing "a table a native query names before sync finds it gets its dependency once a sync of the database finds it"
- (backfill-all-existing-entities!)
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
- "SELECT * FROM PUBLIC.BIRDS_LATE")}
- :model/Card {known-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
- "SELECT * FROM VENUES")}]
- (backfill-all-existing-entities!)
- (is (empty? (table-dependency-ids :card card-id)))
- (is (true? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id card-id)))
- (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id known-id)))
- (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]
- (dependencies.hooks/after-database-sync (mt/id))
- (backfill-all-existing-entities!)
- (is (= #{late-id} (table-dependency-ids :card card-id)))
- (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus
- :entity_type :card :entity_id card-id))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
similarity index 52%
rename from enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
rename to enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
index b07825ca508..d190b2488c1 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
@@ -1,43 +1,549 @@
-(ns metabase-enterprise.dependencies.events-test
+(ns metabase-enterprise.dependencies.task.dependency-test
+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.dependency-test]}}}}}}
(:require
- [clojure.test :refer [deftest is testing]]
- [metabase-enterprise.dependencies.findings :as deps.findings]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [environ.core :as env]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.hooks :as deps.hooks]
- [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as dependencies.model]
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
+ [metabase-enterprise.dependencies.settings :as deps.settings]
+ [metabase-enterprise.dependencies.task.dependency :as task.dependency]
+ [metabase-enterprise.dependencies.task.util :as task.util]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.load :as serdes.load]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
+ [metabase.task.core :as task]
[metabase.test :as mt]
- [toucan2.core :as t2]))
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2])
+ (:import
+ (org.quartz JobKey)))
-(defn- run-with-dependencies-setup! [thunk]
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding :model/DependencyStatus]
- (lib-be/with-metadata-provider-cache
- (let [mp (mt/metadata-provider)]
- (thunk mp)))))))
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(defn- backfill-all-existing-entities!
+ []
+ (deps.test/synchronously-run-dependency-job!))
+
+(defn- backfill-dependencies-single-trigger!
+ []
+ (mt/with-premium-features #{:dependencies}
+ (#'task.dependency/update-dependencies!)))
+
+(defn- forget-status!
+ "Delete the dependency status of an entity, so the dependency job computes its dependencies again."
+ [entity-type entity-id]
+ (dependencies.db/delete-dependency-statuses! entity-type [entity-id]))
-(defn- assert-stale
- "Assert that the given entity is marked stale in dependency_status."
+(def ^:private pending? deps.test/dependency-pending?)
+
+(defn- processed?
+ "Whether the dependency job computed the dependencies of the entity `entity-type` `entity-id` with the current logic
+ and nothing marked them stale since."
[entity-type entity-id]
- (is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)
- (str "Expected " (name entity-type) " " entity-id " to be marked stale")))
+ (and (t2/exists? :model/DependencyStatus
+ :entity_type entity-type
+ :entity_id entity-id
+ :analysis_version dependencies.model/current-dependency-analysis-version)
+ (not (pending? entity-type entity-id))))
(defn- assert-pending
- "Assert that the given entity awaits the backfill job, having no dependency_status row or a stale one."
+ "Assert that the dependency job is to compute the dependencies of the entity `entity-type` `entity-id`."
[entity-type entity-id]
- (is (not (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale false))
+ (is (pending? entity-type entity-id)
(str "Expected " (name entity-type) " " entity-id " to be pending")))
+(defn- assert-processed
+ "Assert that an entity has been processed."
+ [entity-type entity-id]
+ (is (processed? entity-type entity-id)
+ (str "Expected " (name entity-type) " " entity-id " to be processed")))
+
+(deftest ^:synchronized backfill-dependency-analysis-test
+ (testing "the dependency job processes the pending entities in batches, asking to run again while a batch took any"
+ (backfill-all-existing-entities!)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (let [query (mt/mbql-query orders)]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
+ :model/Card {card2-id :id} {:dataset_query query}
+ :model/Card {card3-id :id} {:dataset_query query}]
+ (forget-status! :card card1-id)
+ (forget-status! :card card2-id)
+ (forget-status! :card card3-id)
+ (let [pending-count (fn [] (count (filter #(pending? :card %) [card1-id card2-id card3-id])))]
+ (is (= 3 (pending-count)))
+ ;; first run, should process 2 cards
+ (is (true? (backfill-dependencies-single-trigger!)))
+ (is (= 1 (pending-count)))
+ ;; second run, should process the last card
+ (is (true? (backfill-dependencies-single-trigger!)))
+ (is (= 0 (pending-count)))
+ ;; third run, should not process anything
+ (is (nil? (backfill-dependencies-single-trigger!)))
+ (is (= 0 (pending-count))))))))))
+
+(deftest ^:synchronized backfill-transform-test
+ (testing "Test that transforms are correctly backfilled"
+ (backfill-all-existing-entities!)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Transform {transform-id :id} {:name "Test Transform"
+ :source {:type "query"
+ :query (mt/mbql-query nil
+ {:source-table (str "card__" card-id)})}
+ :target {:type "table" :name (mt/random-name)}}]
+ (forget-status! :card card-id)
+ (forget-status! :transform transform-id)
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (assert-processed :transform transform-id)
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders)))
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :transform :from_entity_id transform-id
+ :to_entity_type :card :to_entity_id card-id)))))))
+
+(deftest ^:synchronized backfill-snippet-test
+ (testing "Test that cards with snippets are correctly backfilled"
+ (backfill-all-existing-entities!)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"
+ :content "SELECT 1"}
+ :model/Card {card-id :id} {:dataset_query {:database (mt/id)
+ :type :native
+ :native {:query "SELECT * FROM {{my_snippet}}"
+ :template-tags
+ {:my_snippet
+ {:id (str (random-uuid))
+ :name "my_snippet"
+ :display-name "my_snippet"
+ :type :snippet
+ :snippet-name "my_snippet"
+ :snippet-id snippet-id}}}}}]
+ (forget-status! :card card-id)
+ (forget-status! :snippet snippet-id)
+ (is (false? (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :snippet :to_entity_id snippet-id)))
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (assert-processed :snippet snippet-id)
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :snippet :to_entity_id snippet-id)))))))
+
+(deftest ^:synchronized backfill-snippet-in-snippet-test
+ (testing "A snippet referencing another snippet produces a snippet->snippet dependency"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner_snip" :content "1 = 1"}
+ :model/NativeQuerySnippet {outer-id :id} {:name "outer_snip" :content "SELECT * WHERE {{snippet: inner_snip}}"}]
+ (forget-status! :snippet inner-id)
+ (forget-status! :snippet outer-id)
+ (is (false? (t2/exists? :model/Dependency
+ :from_entity_type :snippet :from_entity_id outer-id
+ :to_entity_type :snippet :to_entity_id inner-id)))
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :snippet inner-id)
+ (assert-processed :snippet outer-id)
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :snippet :from_entity_id outer-id
+ :to_entity_type :snippet :to_entity_id inner-id))))))
+
+(deftest ^:synchronized backfill-snippet-referencing-card-test
+ (testing "A snippet referencing a card produces a snippet->card dependency"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/NativeQuerySnippet {snippet-id :id} {:name "card_ref_snip"
+ :content (format "SELECT * FROM {{#%d}}" card-id)}]
+ (forget-status! :card card-id)
+ (forget-status! :snippet snippet-id)
+ (is (false? (t2/exists? :model/Dependency
+ :from_entity_type :snippet :from_entity_id snippet-id
+ :to_entity_type :card :to_entity_id card-id)))
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :snippet snippet-id)
+ (is (t2/exists? :model/Dependency
+ :from_entity_type :snippet :from_entity_id snippet-id
+ :to_entity_type :card :to_entity_id card-id))))))
+
+(deftest ^:synchronized backfill-idempotency-test
+ (testing "Running the backfill multiple times should be idempotent"
+ (backfill-all-existing-entities!)
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (forget-status! :card card-id)
+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders))))
+ ;; First run
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ ;; Second run - should not change anything
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders))))))))
+
+(defn- wait-for-condition
+ [predicate timeout-ms]
+ (let [limit (+ (System/currentTimeMillis) timeout-ms)]
+ (loop []
+ (Thread/sleep 100)
+ (or (predicate)
+ (when (< (System/currentTimeMillis) limit)
+ (recur))))))
+
+(deftest ^:synchronized backfill-scheduling-test
+ (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"
+ (backfill-all-existing-entities!)
+ (mt/with-temp-scheduler!
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-batch-size "1"
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")]
+ (let [card-data {:dataset_query (mt/mbql-query orders)}]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card1-id :id} card-data
+ :model/Card {card2-id :id} card-data]
+ (forget-status! :card card1-id)
+ (forget-status! :card card2-id)
+ (let [both-processed? #(every? (partial processed? :card) [card1-id card2-id])]
+ (is (not (both-processed?)))
+ (mt/with-premium-features #{:dependencies}
+ (task/init! ::task.dependency/UpdateDependencies)
+ (wait-for-condition both-processed? 2500)
+ (is (both-processed?))))))))))))
+
+(deftest ^:synchronized backfill-no-trigger-pile-up-test
+ (testing "Scheduling new runs while the dependency job is executing keeps the trigger count ≤ 1"
+ (let [job-started (promise)
+ can-finish (promise)]
+ ;; the job body runs on a Quartz worker thread, which does not inherit *local-redefs* —
+ ;; only a root swap is visible there
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [task.dependency/update-dependencies!
+ (fn []
+ (deliver job-started true)
+ (deref can-finish 1500 :timeout)
+ ;; non-nil → the job will hit the in-job self-reschedule path on exit
+ true)]
+ (mt/with-temp-scheduler!
+ ;; Start the job immediately.
+ (task.util/schedule-run! (task/scheduler) @#'task.dependency/job 0)
+ (is (true? (deref job-started 1000 :timeout))
+ "UpdateDependencies job did not start within 1s")
+ (let [scheduler (task/scheduler)
+ job-key (JobKey. "metabase.task.dependency-backfill.job")
+ trigger-count #(count (.getTriggersOfJob scheduler job-key))]
+ ;; Spam 50 event-driven schedule attempts while the job is in-flight.
+ (dotimes [_ 50]
+ (task.dependency/trigger-dependency-job!))
+ (is (<= (trigger-count) 1)
+ (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))
+ ;; Release the job; on exit it self-reschedules via the in-job path.
+ (deliver can-finish true)
+ (Thread/sleep 200)
+ (is (<= (trigger-count) 1)
+ (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))
+
+(deftest ^:synchronized backfill-error-logging-test
+ (testing "When calculate-deps throws, the error is logged and the entity is computed again on the next run"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (forget-status! :card card-id)
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")
+ deps.calculation/calculate-deps
+ (fn [_ _] (throw (ex-info "Simulated error" {})))]
+ (backfill-dependencies-single-trigger!))
+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))
+ (is (pending? :card card-id))))))
+
+(deftest ^:synchronized backfill-partial-batch-failure-test
+ (testing "A failure on one entity doesn't prevent other entities from being processed"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]
+ (forget-status! :card good-card-id)
+ (forget-status! :card bad-card-id)
+ (let [original-calculate-deps deps.calculation/calculate-deps]
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")
+ deps.calculation/calculate-deps
+ (fn [entity-type entity]
+ (if (= (:id entity) bad-card-id)
+ (throw (ex-info "Simulated error" {}))
+ (original-calculate-deps entity-type entity)))]
+ (backfill-dependencies-single-trigger!)))
+ ;; Good card should be processed successfully
+ (assert-processed :card good-card-id)
+ (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))
+ (is (pending? :card bad-card-id))))))
+
+(deftest ^:synchronized backfill-failure-waits-for-change-test
+ (testing "An entity whose calculation keeps failing is given up on after dependency-max-failures runs, until it changes"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (forget-status! :card card-id)
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps
+ (fn [_ _] (throw (ex-info "Simulated computation error" {:card-id card-id})))]
+ (dotimes [_ (deps.settings/dependency-max-failures)]
+ (backfill-dependencies-single-trigger!)))
+ (is (not (pending? :card card-id)))
+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))
+ (forget-status! :card card-id)
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)))))
+
+(deftest backfill-card-does-not-cause-revision-test
+ (testing "backfilling a card does not create a new revision or audit log entry"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies :audit-app}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (forget-status! :card card-id)
+ (let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)
+ deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
+ (is (= 0 revision-count-before))
+ (is (= 0 deps-before))
+ (backfill-dependencies-single-trigger!)
+ (let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)
+ deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
+ (is (= 0 revision-count-after))
+ (assert-processed :card card-id)
+ (is (= 1 deps-after))))))))
+
+(deftest ^:synchronized backfill-dependencies-on-token-update-test
+ (testing "Token update triggers the dependency job (entities remain pending until job runs)"
+ (backfill-all-existing-entities!)
+ (let [query (mt/mbql-query orders)]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
+ :model/Card {card2-id :id} {:dataset_query query}
+ :model/Card {card3-id :id} {:dataset_query query}]
+ (forget-status! :card card1-id)
+ (forget-status! :card card2-id)
+ (forget-status! :card card3-id)
+ ;; token update triggers the async job, not synchronous processing
+ (dependencies.hooks/after-premium-token-update)
+ (is (every? #(pending? :card %) [card1-id card2-id card3-id]))
+ ;; manually run the backfill to verify it processes them
+ (backfill-all-existing-entities!)
+ (is (every? #(processed? :card %) [card1-id card2-id card3-id])))))))
+
+(deftest ^:synchronized backfill-version-outdated-test
+ (testing "Entities with outdated version in dependency_status get reprocessed"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (backfill-dependencies-single-trigger!)
+ (t2/update! :model/DependencyStatus :entity_type :card :entity_id card-id {:analysis_version 0})
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))
+
+(deftest ^:synchronized backfill-no-status-row-test
+ (testing "Entities with no dependency_status row yet get picked up and processed by the backfill"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))
+ "Expected the temp card to start with no dependency_status row")
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))
+
+(deftest ^:synchronized has-pending-entities?-counts-no-status-row-test
+ (testing "has-pending-entities? (backing /backfill-status) stays consistent with what the dependency job processes"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ ;; Clean slate: nothing left to process.
+ (is (false? (deps.dependency-status/has-pending-entities?))
+ "Expected no pending work after backfilling all existing entities")
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ ;; A card with no status row is pending work, even though no DependencyStatus row exists.
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id)))
+ (is (true? (deps.dependency-status/has-pending-entities?))
+ "Expected pending work: a card with no status row still needs calculation")
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (is (false? (deps.dependency-status/has-pending-entities?))
+ "Expected no pending work once the card has been processed")))))
+
+(deftest ^:synchronized batch-size-zero-disables-job-test
+ (testing "a batch size that isn't positive keeps the job from being scheduled by writes or at startup"
+ (let [scheduled (atom [])
+ scheduled-by (fn [thunk]
+ (reset! scheduled [])
+ (thunk)
+ (count @scheduled))]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]
+ (when (some #(str/includes? (str %) "dependency-backfill") args)
+ (swap! scheduled conj args))
+ nil)]
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")]
+ (is (zero? (scheduled-by task.dependency/trigger-dependency-job!)))
+ (is (zero? (scheduled-by #(task/init! ::task.dependency/UpdateDependencies)))))
+ (testing "a positive batch size schedules from the event trigger"
+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "5")]
+ (is (= 1 (scheduled-by task.dependency/trigger-dependency-job!)))))))))
+
+(deftest ^:synchronized backfill-batch-shares-one-metadata-provider-test
+ (testing "GHY-4251: every entity in a batch must share one MetadataProvider per database. Reading an entity attaches
+ a provider to its query, so without a batch-scoped cache each entity gets a private one; native dep
+ calculation then fills each with that database's table metadata and the batch retains one copy per entity,
+ which is what exhausted the heap on instances with large warehouses."
+ (backfill-all-existing-entities!)
+ (let [providers (atom [])
+ calculate-deps (mt/original-fn #'deps.calculation/calculate-deps)
+ native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}
+ :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]
+ (forget-status! :card card1-id)
+ (forget-status! :card card2-id)
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps
+ (fn [entity-type entity]
+ (when-let [mp (:lib/metadata (:dataset_query entity))]
+ (swap! providers conj mp))
+ (calculate-deps entity-type entity))]
+ (backfill-dependencies-single-trigger!))
+ (testing "both cards were analyzed, so the assertion below is meaningful"
+ (is (<= 2 (count @providers))))
+ (testing "and they were analyzed through the same provider instance"
+ ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be
+ ;; satisfied by two separate caches and would not detect the regression.
+ (is (= 1 (count (into #{} (map #(System/identityHashCode %)) @providers))))))))))
+
+(deftest ^:synchronized fatal-error-gives-up-after-max-failures-test
+ (testing "GHY-4251: a fatal Error propagates and counts as a failure, so an entity that keeps causing one is given up
+ on after dependency-max-failures runs instead of replaying the batch in a crash loop"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (forget-status! :card card-id)
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (Error. "simulated fatal error")))]
+ (dotimes [run (deps.settings/dependency-max-failures)]
+ (testing (str "run " (inc run) ": the Error propagates instead of being swallowed")
+ (is (thrown-with-msg? Error #"simulated fatal error" (backfill-dependencies-single-trigger!))))
+ (testing (str "run " (inc run) ": the entity stays pending until the last allowed failure")
+ (is (= (< (inc run) (deps.settings/dependency-max-failures)) (pending? :card card-id))))))
+ (testing "a write gives the entity fresh attempts"
+ (dependencies.db/mark-dependency-statuses-stale! {:card #{card-id}})
+ (is (pending? :card card-id)))))))
+
+(deftest ^:synchronized failing-entity-gives-up-after-max-failures-test
+ (testing "an entity with no dependency_status row yet whose computation keeps failing is retried until
+ dependency-max-failures, then given up on until it changes"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))
+ "test setup: the card must have no status row, which is what makes it eligible")
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (ex-info "boom" {})))]
+ (backfill-dependencies-single-trigger!)
+ (is (pending? :card card-id))
+ (dotimes [_ (dec (deps.settings/dependency-max-failures))]
+ (backfill-dependencies-single-trigger!)))
+ (is (not (pending? :card card-id)))
+ (is (=? {:terminal true :fail_count (deps.settings/dependency-max-failures)}
+ (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)))))))
+
+(deftest ^:synchronized fatal-error-still-reschedules-test
+ (testing "A failure must not leave the job unscheduled. The job is one-shot self-rescheduling with no cron
+ backstop, so an Error propagating out of the batch skipped schedule-run! entirely and the chain ended
+ until a content-change event or a restart. A real OutOfMemoryError usually takes the process with it and
+ task/init! recovers on boot, but survivable Errors -- StackOverflowError from deeply nested SQL, an
+ AssertionError, a LinkageError -- leave a live process with a dead job."
+ (let [scheduled (atom [])]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)
+ task.dependency/update-dependencies!
+ (fn [& _] (throw (Error. "simulated fatal error")))]
+ (testing "the Error still propagates"
+ (is (thrown-with-msg? Error #"simulated fatal error"
+ (task.util/run-and-reschedule! nil @#'task.dependency/job))))
+ (testing "and the next run was scheduled anyway"
+ (is (= 1 (count @scheduled))))))))
+
+(defn- table-dependency-ids
+ "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."
+ [entity-type entity-id]
+ (t2/select-fn-set :to_entity_id :model/Dependency
+ :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))
+
+(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test
+ (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,
+ recomputes the cards whose SQL expands it"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}
+ :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}
+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "{{snippet: birds outer}}")}]
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))
+ (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))
+
+(deftest ^:synchronized snippet-new-upstream-marks-analysis-test
+ (testing "storing a new upstream of a snippet marks the analysis of what expands it, which an edit of that upstream
+ before the edge existed never marked"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "window inner" :content "VENUES"}
+ :model/NativeQuerySnippet {outer-id :id} {:name "window outer" :content "SELECT * FROM VENUES"}
+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "{{snippet: window outer}}")}]
+ (backfill-all-existing-entities!)
+ (t2/update! :model/NativeQuerySnippet outer-id {:content "SELECT * FROM {{snippet: window inner}}"})
+ (deps.test/record-analysis-finding-processed! :card card-id)
+ (backfill-dependencies-single-trigger!)
+ (is (deps.test/analysis-finding-pending? :card card-id))))))
+
+(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test
+ (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
+ :model/Card {card-id :id} {:dataset_query
+ (lib/native-query
+ mp
+ (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"
+ orders-id))}]
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :card :to_entity_id orders-id)))))))
+
(deftest dashboard-update-sets-correct-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [products-id (mt/id :products)
orders-id (mt/id :orders)
@@ -59,7 +565,7 @@
:card_id series-card-id
:position 0}]
(assert-pending :dashboard dashboard-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :dashboard
:from_entity_id dashboard-id
:to_entity_type :card
@@ -114,7 +620,7 @@
:visualization_settings {:column_settings {"[\"name\",\"CATEGORY\"]" {:click_behavior {:type "link"
:linkType "dashboard"
:targetId column-click-target-dashboard-id}}}}}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :dashboard
:from_entity_id dashboard-id
:to_entity_type :card
@@ -150,11 +656,11 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))
(t2/delete! :model/Dashboard dashboard-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))
(deftest document-update-sets-correct-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
@@ -171,7 +677,7 @@
{:type "cardEmbed"
:attrs {"id" embedded-card-id}}]}}]
(assert-pending :document document-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
:to_entity_type :card
@@ -191,7 +697,7 @@
:attrs {"entityId" products-id
"model" "table"}}]}]})]
(t2/update! :model/Document document-id updated-doc)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
:to_entity_type :dashboard
@@ -203,12 +709,12 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))
(t2/delete! :model/Document document-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))))))))
(deftest sandbox-update-sets-correct-dependencies
(mt/with-premium-features #{:sandboxes}
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "sandbox group"}
:model/Card {card1-id :id} {}
@@ -217,7 +723,7 @@
:table_id (mt/id :products)
:card_id card1-id}]
(assert-pending :sandbox sandbox-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :sandbox
:from_entity_id sandbox-id
:to_entity_type :card
@@ -225,7 +731,7 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))
(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? #{{:from_entity_type :sandbox
:from_entity_id sandbox-id
:to_entity_type :card
@@ -233,30 +739,27 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))
(t2/delete! :model/Sandbox sandbox-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))
(deftest card-marks-pending-on-create-and-stale-on-update-test
- (testing "A created card is pending for the backfill job, and changing its query marks it stale in dependency_status"
- (run-with-dependencies-setup!
+ (testing "a created card is pending for the dependency job, and any update makes it pending again"
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [products (lib.metadata/table mp (mt/id :products))
orders (lib.metadata/table mp (mt/id :orders))]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
(assert-pending :card card-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :products)))
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false))
- (testing "a change to a column the dependencies do not come from leaves it up to date"
- (t2/update! :model/Card card-id {:name "renamed"})
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false)))
+ (is (not (pending? :card card-id)))
(t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)})
- (assert-stale :card card-id)))))))
+ (assert-pending :card card-id)))))))
(deftest metric-dimensions-update-marks-stale-and-includes-mapped-table-test
(testing "a metric dimension write marks the metric card stale and recomputes deps, including mapped-dimension tables"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [base (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
(lib/aggregate (lib/count)))]
@@ -270,8 +773,8 @@
:dimension-id "550e8400-e29b-41d4-a716-446655440000"
:table-id (mt/id :categories)
:target [:field {} (mt/id :categories :name)]}]})
- (assert-stale :card card-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :card card-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :categories))
"the mapped dimension's table becomes a dependency")
@@ -280,41 +783,26 @@
"the query's own table remains a dependency")))))))
(deftest snippet-marks-pending-on-create-and-stale-on-update-test
- (testing "A created snippet is pending for the backfill job, and changing its content marks it stale in dependency_status"
- (run-with-dependencies-setup!
+ (testing "a created snippet is pending for the dependency job, and changing its content makes it pending again"
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "test snippet"
:content "SELECT 1"}]
(assert-pending :snippet snippet-id)
- (deps.test/synchronously-run-backfill!)
- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id snippet-id :stale false))
+ (deps.test/synchronously-run-dependency-job!)
+ (is (not (pending? :snippet snippet-id)))
(t2/update! :model/NativeQuerySnippet snippet-id {:content "SELECT 2"})
- (assert-stale :snippet snippet-id))))))
-
-(deftest mark-stale-failure-fails-the-write-test
- (testing "When mark-stale! throws, the update fails and the card keeps its query"
- (run-with-dependencies-setup!
- (fn [mp]
- (let [products (lib.metadata/table mp (mt/id :products))
- orders (lib.metadata/table mp (mt/id :orders))]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
- (deps.test/synchronously-run-backfill!)
- (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!
- (fn [_] (throw (ex-info "Simulated DB failure" {})))]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated DB failure"
- (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)}))))
- (is (=? {:stages [{:source-table (mt/id :products)}]}
- (t2/select-one-fn :dataset_query :model/Card card-id)))))))))
+ (assert-pending :snippet snippet-id))))))
(deftest ^:synchronized native-transform-create-sets-dependencies-test
(testing "a created native transform is picked up by the dependency calculation"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}]
(mt/with-temp [:model/Transform {transform-id :id} {:source source}]
(assert-pending :transform transform-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
@@ -323,13 +811,13 @@
(deftest ^:synchronized mbql-transform-create-sets-dependencies-test
(testing "a created mbql transform is picked up by the dependency calculation"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
:type :query}]
(mt/with-temp [:model/Transform {transform-id :id} {:source source}]
(assert-pending :transform transform-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
@@ -338,7 +826,7 @@
(deftest ^:synchronized python-transform-create-sets-dependencies-test
(testing "a created python transform is picked up by the dependency calculation"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
:source-database (mt/id),
@@ -351,61 +839,16 @@
"import pandas as pd\n\ndef transform(orders):\n return orders"}]
(mt/with-temp [:model/Transform {transform-id :id} {:source source}]
(assert-pending :transform transform-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
:to_entity_id (mt/id :orders)}]
(t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
-(deftest ^:synchronized database-delete-marks-source-transforms-stale-test
- (testing "deleting a database marks its source transforms' analysis findings stale so they surface on /dependency-diagnostics/broken (GDGT-2447)"
- (run-with-dependencies-setup!
- (fn [_mp]
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Transform {transform-id :id}
- {:source {:type :query
- :query {:database db-id
- :type :native
- :native {:query "SELECT 1"}}}
- :target {:schema "public" :name "out" :type :table}}
- :model/AnalysisFinding _ {:analyzed_entity_type :transform
- :analyzed_entity_id transform-id
- :result true
- :stale false
- :analysis_version models.analysis-finding/*current-analysis-finding-version*
- :analyzed_at :%now}]
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))
- "baseline: existing finding is fresh")
- (t2/delete! :model/Database db-id)
- (is (nil? (t2/select-one :model/Database db-id))
- "the Database row is actually gone")
- (assert-stale :transform transform-id)
- (deps.test/synchronously-run-backfill!)
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))
- "after db-delete the backfill marks the transform's finding stale")
- (testing "and re-running the entity-check job flips :result false so the transform shows on broken-diagnostics"
- (#'task.entity-check/check-entities!)
- (let [{:keys [stale result]} (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id)]
- (is (false? stale))
- (is (false? result)))
- (testing "an analysis_finding_error row points the transform at itself so the diagnostics page (`/graph/breaking`) surfaces it"
- (is (=? {:source_entity_type :transform
- :source_entity_id transform-id
- :error_type :validation-exception-error}
- (t2/select-one :model/AnalysisFindingError
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))))))))))
-
(deftest ^:synchronized transform-run-updates-dependencies-test
(testing "transform run events trigger dependency calculations"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
@@ -419,7 +862,7 @@
(deftest ^:synchronized python-transform-update-handles-downstream-dependencies-test
(testing "python transform updates handle downstream dependencies"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
:source-database (mt/id),
@@ -444,8 +887,8 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "keeping target"
(t2/update! :model/Transform transform-id {:source (update source :body str "\n")})
- (assert-stale :transform transform-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :transform transform-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
@@ -453,13 +896,13 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty?
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
(deftest ^:synchronized query-transform-update-handles-downstream-dependencies-test
(testing "query transform updates handle downstream dependencies"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}
@@ -476,8 +919,8 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "keeping target"
(t2/update! :model/Transform transform-id {:source (assoc source :query (lib/native-query mp "select * from orders limit 10"))})
- (assert-stale :transform transform-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :transform transform-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
@@ -485,12 +928,12 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty?
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
(deftest segment-update-sets-correct-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)
@@ -499,7 +942,7 @@
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a segment creates dependency to its table"
(assert-pending :segment segment-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :segment
:from_entity_id segment-id
:to_entity_type :table
@@ -508,8 +951,8 @@
(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))
(testing "updating segment definition recalculates dependencies"
(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})
- (assert-stale :segment segment-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :segment segment-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :segment
:from_entity_id segment-id
:to_entity_type :table
@@ -518,23 +961,23 @@
(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))
(testing "deleting segment removes all dependencies"
(t2/delete! :model/Segment segment-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))
(deftest card-with-segment-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
(mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a card using a segment creates dependencies to both segment and table"
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(mt/with-temp [:model/Card {card-id :id} {:dataset_query {:database (mt/id)
:type :query
:query {:source-table products-id
:filter [:segment segment-id]}}}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
:to_entity_type :segment
@@ -547,7 +990,7 @@
(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))
(deftest measure-update-sets-correct-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(mt/with-test-user :rasta
(let [orders-id (mt/id :orders)
@@ -559,7 +1002,7 @@
(lib/aggregate (lib/sum quantity)))}]
(testing "creating a measure creates dependency to its table"
(assert-pending :measure measure-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure-id
:to_entity_type :table
@@ -569,8 +1012,8 @@
(testing "updating measure definition recalculates dependencies"
(t2/update! :model/Measure measure-id {:definition (-> (lib/query mp orders)
(lib/aggregate (lib/sum subtotal)))})
- (assert-stale :measure measure-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :measure measure-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure-id
:to_entity_type :table
@@ -579,11 +1022,11 @@
(t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))
(testing "deleting measure removes all dependencies"
(t2/delete! :model/Measure measure-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))
(deftest action-update-sets-correct-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [action-deps (fn [action-id]
(t2/select-fn-set (juxt :to_entity_type :to_entity_id)
@@ -594,19 +1037,19 @@
:database_id (mt/id)
:dataset_query (sql-query "venues")}]
(testing "creating an action creates dependencies on the tables its query reads"
- (assert-stale :action action-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :action action-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{[:table (mt/id :venues)]}
(action-deps action-id))))
(testing "updating an action's query recalculates its dependencies"
(t2/update! :model/QueryAction action-id {:dataset_query (sql-query "checkins")})
- (assert-stale :action action-id)
- (deps.test/synchronously-run-backfill!)
+ (assert-pending :action action-id)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{[:table (mt/id :checkins)]}
(action-deps action-id))))
(testing "deleting an action removes its dependencies and status"
(t2/delete! :model/Action action-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (empty? (action-deps action-id)))
(is (not (t2/exists? :model/DependencyStatus :entity_type :action :entity_id action-id)))))))))
@@ -619,7 +1062,7 @@
:entity_type :action :entity_id [:in action-ids]))})
(deftest model-delete-removes-dependency-rows-of-its-actions-test
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(mt/with-temp [:model/Card {model-id :id} {:type :model
:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}
@@ -629,18 +1072,18 @@
:model/QueryAction _ {:action_id query-id
:database_id (mt/id)
:dataset_query (lib/native-query mp "DELETE FROM venues WHERE id = 1")}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "the model's actions have dependency rows"
(is (= {:dependencies #{implicit-id query-id}, :statuses #{implicit-id query-id}}
(action-dependency-rows [implicit-id query-id]))))
(testing "deleting the model removes the dependency rows of its actions"
(t2/delete! :model/Card model-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= {:dependencies #{}, :statuses #{}}
(action-dependency-rows [implicit-id query-id]))))))))
(deftest database-delete-removes-dependency-rows-of-its-models-actions-test
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [_mp]
(mt/with-temp [:model/Database {db-id :id} {:engine :h2, :details (:details (mt/db))}
:model/Table {table-id :id} {:db_id db-id, :name "VENUES", :schema "PUBLIC"}]
@@ -654,19 +1097,19 @@
:model/QueryAction _ {:action_id query-id
:database_id db-id
:dataset_query (lib/native-query db-mp "DELETE FROM venues WHERE id = 1")}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "the actions of the database's model have status rows"
(is (= #{implicit-id query-id}
(:statuses (action-dependency-rows [implicit-id query-id])))))
(testing "deleting the database removes the dependency rows of the actions of its models"
(t2/delete! :model/Database db-id)
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= {:dependencies #{}, :statuses #{}}
(action-dependency-rows [implicit-id query-id]))))
(is (some? table-id))))))))
(deftest measure-with-measure-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
@@ -681,7 +1124,7 @@
(lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)
(lib/sum subtotal))))}]
(testing "creating a measure that references another measure creates dependencies to both"
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure2-id
:to_entity_type :table
@@ -694,7 +1137,7 @@
(t2/select :model/Dependency :from_entity_id measure2-id :from_entity_type :measure)))))))))))))
(deftest card-with-measure-dependencies
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
@@ -702,13 +1145,13 @@
(mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id
:definition (-> (lib/query mp orders)
(lib/aggregate (lib/sum quantity)))}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "creating a card using a measure creates dependencies to both measure and table"
(let [mp' (mt/metadata-provider)
query (-> (lib/query mp' orders)
(lib/aggregate (lib.metadata/measure mp' measure-id)))]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
:to_entity_type :measure
@@ -720,388 +1163,37 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card)))))))))))))
-;; === Analysis finding tests (these are unchanged — they use AnalysisFinding, not dependency_analysis_version) ===
-
-(defn- assert-has-analyses
- "Asserts that a given set of entities have the appropriate analysis version.
-
- Format is `{entity-type {entity-id analysis-version}}`.
-
- Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version.
-
- Use nil for analysis-version if you want to check that no analysis exists."
- [spec]
- (doseq [[entity-type ids-and-versions] spec]
- (let [analyses-map (->> (t2/select [:model/AnalysisFinding :analyzed_entity_id :analysis_version]
- :analyzed_entity_type entity-type)
- (into {} (map (juxt :analyzed_entity_id :analysis_version))))]
- (doseq [[id expected-version] ids-and-versions]
- (testing (str "Checking " entity-type " " id)
- (if (= expected-version -1)
- (is (contains? analyses-map id))
- (is (= expected-version (analyses-map id)))))))))
-
-(deftest ^:synchronized card-update-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Card updates should mark the card stale and trigger the background job"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
- :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
- :to_entity_type :card
- :to_entity_id parent-card-id}]
- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})
- ;; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities
- (assert-has-analyses
- {:card {parent-card-id nil}})
- (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-card-id))
- "entities without analysis findings should not be marked: no fake analysis record should be created")))))))
-
-(deftest ^:synchronized card-update-updates-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Card updates should mark the card stale, then the job re-analyzes and propagates"
- (let [products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
- :to_entity_type :card
- :to_entity_id parent-card-id}]
- (deps.findings/upsert-analysis! parent-card)
- (deps.findings/upsert-analysis! child-card)
- (deps.findings/upsert-analysis! other-card)
- (deps.dependency-status/upsert-status! :card child-card-id false)
- (deps.dependency-status/upsert-status! :card other-card-id false)
- (testing "Checking that the initial analyses exist"
- (assert-has-analyses
- {:card {parent-card-id old-version
- child-card-id old-version
- other-card-id old-version}}))
- (testing "changing the card query marks its finding and its dependents' stale, not unrelated ones"
- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
- (deps.test/synchronously-run-backfill!)
- (assert-has-analyses
- {:card {parent-card-id old-version
- child-card-id -1
- other-card-id old-version}})
- (is (= {parent-card-id true child-card-id true other-card-id false}
- (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding :analyzed_entity_type :card
- :analyzed_entity_id [:in [parent-card-id child-card-id other-card-id]]))))
- (testing "If the version changes, the job picks up only the stale cards"
- (binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)]
- (assert-has-analyses
- {:card {parent-card-id old-version
- child-card-id -1
- other-card-id old-version}})))))))))
-
-;;; ------------------------------------------------ Analysis propagation tests ------------------------------------------------
-
-(deftest ^:synchronized card-update-triggers-native-cards-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Card update marks the card and its transitive dependents stale, and the entity-check job re-analyzes them"
- (mt/with-temp [:model/Card {parent-id :id :as parent} {:dataset_query (lib/native-query mp "select * from products")}
- :model/Card {child-id :id :as child} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
- :model/Dependency _ {:from_entity_type :card :from_entity_id child-id
- :to_entity_type :card :to_entity_id parent-id}]
- (deps.findings/upsert-analysis! parent)
- (deps.findings/upsert-analysis! child)
- (deps.dependency-status/upsert-status! :card child-id false)
- (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})
- (deps.test/synchronously-run-backfill!)
- (testing "Parent card should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id parent-id))))
- ;; Run entity-check job to process stale entities
- (#'task.entity-check/check-entities!)
- (testing "Parent should be re-analyzed"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id parent-id))))
- (testing "Child should have been re-analyzed (it was marked stale up front as a transitive dependent)"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-id)))))))))
-
-(deftest ^:synchronized card-update-stops-on-transforms-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Card update marks the card and its transitive dependents (including transforms) stale up front"
- (let [products (lib.metadata/table mp (mt/id :products))
- orders (lib.metadata/table mp (mt/id :orders))
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :transform :from_entity_id transform-id
- :to_entity_type :card :to_entity_id parent-card-id}
- :model/Dependency _ {:from_entity_type :card :from_entity_id child-card-id
- :to_entity_type :transform :to_entity_id transform-id}]
- (deps.findings/upsert-analysis! parent-card)
- (deps.findings/upsert-analysis! transform)
- (deps.findings/upsert-analysis! child-card)
- (assert-has-analyses
- {:card {parent-card-id old-version child-card-id old-version}
- :transform {transform-id old-version}})
- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
- ;; Run entity-check job to drain the stale entities
- (#'task.entity-check/check-entities!)
- (testing "All entities should be re-analyzed after the drain"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id parent-card-id)))
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform :analyzed_entity_id transform-id)))
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id child-card-id))))))))))
-
-(deftest ^:synchronized transform-update-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Transform update does not error when no analyses exist yet"
- (let [products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- (is (= 1 (t2/update! :model/Transform transform-id
- {:source {:type :query
- :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))
-
-(deftest ^:synchronized transform-update-updates-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Transform update marks entity stale, entity-check job re-analyzes it"
- (let [products (lib.metadata/table mp (mt/id :products))
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- (deps.findings/upsert-analysis! transform)
- (assert-has-analyses {:transform {transform-id old-version}})
- (t2/update! :model/Transform transform-id {:source {:type :query
- :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})
- (deps.test/synchronously-run-backfill!)
- (testing "Transform should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))))
- (#'task.entity-check/check-entities!)
- (testing "Transform should be re-analyzed"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))))))))))
-
-(deftest ^:synchronized transform-update-triggers-native-transforms-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Native transform updates do not error"
- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
- :query (lib/native-query mp "select * from products")}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- (is (= 1 (t2/update! :model/Transform transform-id
- {:source {:type :query
- :query (lib/native-query mp "select * from orders")}}))))))))
-
-(deftest ^:synchronized transform-run-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Transform run completes without error when no analyses exist"
- (let [products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public" :name "sample" :type :table}}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :transform :to_entity_id transform-id}]
- (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))
-
-(deftest ^:synchronized transform-run-marks-dependents-stale-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Transform run marks immediate dependents stale"
- (let [products (lib.metadata/table mp (mt/id :products))
- orders (lib.metadata/table mp (mt/id :orders))]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id :as transform} {:source {:type :query :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public" :name "sample" :type :table}}
- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :transform :to_entity_id transform-id}]
- (deps.findings/upsert-analysis! card)
- (deps.findings/upsert-analysis! transform)
- (deps.findings/upsert-analysis! other-card)
- (deps.hooks/after-transform-run transform-id nil)
- (testing "Dependent card should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))
- (testing "Other card should be unchanged"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))
- (testing "Transform should be unchanged"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :transform :analyzed_entity_id transform-id))))))))))
-
-(deftest ^:synchronized segment-update-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [_]
- (testing "Segment update does not error when no analyses exist yet"
- (let [products-id (mt/id :products)
- price-field-id (mt/id :products :price)]
- (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}]
- (is (= 1 (t2/update! :model/Segment segment-id
- {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))
-
-(deftest ^:synchronized segment-update-updates-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Segment update marks entity stale, entity-check job re-analyzes and propagates"
- (let [products-id (mt/id :products)
- price-field-id (mt/id :products :price)
- products (lib.metadata/table mp products-id)
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}
- :model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :segment :to_entity_id segment-id}]
- (deps.findings/upsert-analysis! segment)
- (deps.findings/upsert-analysis! card)
- (deps.dependency-status/upsert-status! :card card-id false)
- (assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})
- (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})
- (deps.test/synchronously-run-backfill!)
- (testing "Segment should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :segment :analyzed_entity_id segment-id))))
- (#'task.entity-check/check-entities!)
- (testing "Both segment and dependent card should be re-analyzed"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :segment :analyzed_entity_id segment-id)))
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))))))))
-
-(deftest ^:synchronized table-metadata-update-triggers-dependent-analysis-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Table metadata update marks immediate dependents stale"
- (let [products-id (mt/id :products)
- orders (lib.metadata/table mp (mt/id :orders))
- products (lib.metadata/table mp products-id)
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id products-id}]
- (deps.findings/upsert-analysis! card)
- (deps.findings/upsert-analysis! other-card)
- (assert-has-analyses {:card {card-id old-version other-card-id old-version}})
- (deps.test/do-with-immediate-hook-writes
- #(mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
- (testing "Card depending on updated table should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))
- (testing "Card not depending on updated table should not be stale"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))
-
-(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Table metadata update does not error when no findings exist yet"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id products-id}]
- (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
- (assert-has-analyses {:card {card-id nil}}))))))))
-
-(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Field metadata update marks dependents of the field's table stale"
- (let [products-id (mt/id :products)
- orders (lib.metadata/table mp (mt/id :orders))
- products (lib.metadata/table mp products-id)
- old-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id products-id}]
- (deps.findings/upsert-analysis! card)
- (deps.findings/upsert-analysis! other-card)
- (assert-has-analyses {:card {card-id old-version other-card-id old-version}})
- (deps.test/do-with-immediate-hook-writes
- #(mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]
- (testing "Card depending on the table whose field was updated should be stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))
- (testing "Card not depending on that table should not be stale"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))
-
-(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Field metadata update does not error when no findings exist yet"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
- :to_entity_type :table :to_entity_id products-id}]
- (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]
- (assert-has-analyses {:card {card-id nil}}))))))))
-
(deftest serdes-load-recalculates-changed-entities-test
(testing "a serdes load recalculates the dependencies of the entities it changes, and only those"
- (run-with-dependencies-setup!
+ (deps.test/run-with-dependencies-setup!
(fn [mp]
(let [card-deps #(t2/select-fn-set (juxt :to_entity_type :to_entity_id)
:model/Dependency :from_entity_type :card :from_entity_id %)
query #(lib/query mp (lib.metadata/table mp (mt/id %)))
- processed #(t2/exists? :model/DependencyStatus :entity_type :card :entity_id % :stale false)
load! #(serdes/with-cache
(serdes.load/load-metabase! (ts/ingestion-in-memory %) :reindex? false))]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (query :products)}
:model/Card {other-id :id} {:dataset_query (query :products)}]
(let [exported (serdes/extract-one "Card" {} card)]
(t2/update! :model/Card card-id {:dataset_query (query :orders)})
- (deps.dependency-status/mark-stale! {:card #{card-id other-id}})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (= #{[:table (mt/id :orders)]} (card-deps card-id)))
(load! [exported])
- (testing "only the changed card is marked stale"
- (assert-stale :card card-id)
- (is (processed other-id)))
- (deps.test/synchronously-run-backfill!)
- (is (= #{[:table (mt/id :products)]} (card-deps card-id)))
- (testing "reloading it unchanged marks nothing"
- (load! [exported])
- (is (processed card-id))))))))))
+ (testing "only the changed card is pending"
+ (assert-pending :card card-id)
+ (is (not (pending? :card other-id))))
+ (deps.test/synchronously-run-dependency-job!)
+ (is (= #{[:table (mt/id :products)]} (card-deps card-id))))))))))
+
+(deftest ^:synchronized new-upstream-marks-analysis-test
+ (testing "storing a new upstream of an entity marks its analysis stale, since one done before the edge missed the
+ upstream's changes"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFindingStatus :model/DependencyStatus]
+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (deps.test/table-query :orders)}
+ :model/Card {card-id :id} {:dataset_query (deps.test/table-query :venues)}]
+ (deps.test/synchronously-run-dependency-job!)
+ (t2/update! :model/Card card-id {:dataset_query (deps.test/card-query source-id)})
+ (deps.test/record-analysis-finding-processed! :card card-id)
+ (deps.test/synchronously-run-dependency-job!)
+ (is (deps.test/analysis-finding-pending? :card card-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj
deleted file mode 100644
index aa770887e24..00000000000
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj
+++ /dev/null
@@ -1,75 +0,0 @@
-(ns metabase-enterprise.dependencies.task.entity-check-test
- (:require
- [clojure.test :refer [deftest is testing]]
- [environ.core :as env]
- [metabase-enterprise.dependencies.task.entity-check :as dependencies.entity-check]
- [metabase.dependencies.hooks :as dependencies.hooks]
- [metabase.premium-features.core :as premium-features]
- [metabase.task.core :as task]
- [metabase.test :as mt])
- (:import
- (org.quartz JobDetail)))
-
-(set! *warn-on-reflection* true)
-
-(deftest ^:synchronized batch-size-zero-suppresses-event-triggers-but-stays-warm-test
- (testing "A non-positive batch size stops the job doing work, but leaves it on its slow periodic schedule so it
- resumes if the setting ever becomes positive. What it must not do is keep firing the 1-second event-driven
- trigger: entity changes fire it, and it never consulted the batch size, so a disabled job woke roughly once
- a second on a busy instance."
- (let [scheduled (atom [])
- scheduled-by (fn [thunk]
- (reset! scheduled [])
- (thunk)
- (count @scheduled))]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
- (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "0")]
- (testing "the 1-second event-driven trigger is suppressed"
- (is (zero? (scheduled-by dependencies.entity-check/trigger-entity-check-job!))))
- (testing "but task/init! still puts the job on its periodic schedule"
- (is (= 1 (scheduled-by #(task/init! ::dependencies.entity-check/DependencyEntityCheck)))))
- (testing "and a run keeps the periodic chain going"
- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly true)]
- (is (= 1 (scheduled-by #(#'dependencies.entity-check/reschedule-after-run! nil)))))))
- (testing "a positive batch size schedules from the event trigger too"
- (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "5")]
- (is (= 1 (scheduled-by dependencies.entity-check/trigger-entity-check-job!)))))))))
-
-(deftest ^:synchronized post-run-reschedule-licence-states-test
- (testing "The entity check job is a periodic checker with no terminal state, so it always queues another run. That
- is right while licensed, but check-entities! is gated on :dependencies, so without the feature the job
- rescheduled itself every 30-90 minutes forever to do nothing at all.
-
- Scheduling resolves the licence with canonically-has-feature?, not has-feature?, so it can distinguish
- 'definitively unlicensed' from 'could not tell'. has-feature? collapses both to false, and since nothing
- re-fires when a failed token check later succeeds, reading a blip as unlicensed would end the chain for
- good. Only a definitive false stops it."
- (let [scheduled (atom [])
- scheduled-by (fn [licence]
- (reset! scheduled [])
- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly licence)]
- (#'dependencies.entity-check/reschedule-after-run! nil))
- (count @scheduled))]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
- (testing "licensed: keeps itself scheduled"
- (is (= 1 (scheduled-by true))))
- (testing "indeterminate: keeps the chain alive rather than ending it on a transient failure"
- (is (= 1 (scheduled-by nil))))
- (testing "definitively unlicensed: stops"
- (is (zero? (scheduled-by false))))))))
-
-(def ^:private entity-check-job-name "metabase.dependencies.task.entity-check.job")
-
-(defn- scheduled-job-names
- "Job names from captured [[task/schedule-task!]] calls, since a token update schedules both dependency jobs."
- [calls]
- (into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))
-
-(deftest ^:synchronized token-update-restarts-the-job-test
- (testing "Setting a premium token must restart the entity check job, or enabling the feature on an idle instance
- leaves the periodic checker dead until the next process restart."
- (let [scheduled (atom [])]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
- (mt/with-premium-features #{:dependencies}
- (dependencies.hooks/after-premium-token-update)
- (is (contains? (scheduled-job-names @scheduled) entity-check-job-name)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.clj
new file mode 100644
index 00000000000..d1325645624
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.dependencies.task.util-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]
+ [metabase-enterprise.dependencies.task.util :as task.util]
+ [metabase.task.core :as task]
+ [metabase.test :as mt])
+ (:import
+ (java.util Date)
+ (org.quartz Scheduler TriggerKey)))
+
+(set! *warn-on-reflection* true)
+
+(defn- seconds-until-next-run ^long []
+ (let [fire-time (.getNextFireTime (.getTrigger ^Scheduler (task/scheduler)
+ (TriggerKey. "metabase.dependencies.task.result-metadata.trigger")))]
+ (quot (- (.getTime ^Date fire-time) (System/currentTimeMillis)) 1000)))
+
+(deftest schedule-run-keeps-earlier-run-test
+ (testing "scheduling a later run keeps a run already pending earlier, and an earlier run replaces a later one"
+ (mt/with-temp-scheduler!
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600)
+ (is (< 3000 (seconds-until-next-run)))
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)
+ (is (< (seconds-until-next-run) 61))
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600)
+ (is (< (seconds-until-next-run) 61)))))
+
+(deftest schedule-run-leaves-earlier-run-alone-test
+ (testing "scheduling a later run while an earlier one is pending does not replace the pending trigger"
+ (mt/with-temp-scheduler!
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)
+ (let [scheduled (atom 0)]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& _] (swap! scheduled inc))]
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600))
+ (is (zero? @scheduled))))))
+
+(deftest schedule-run-moves-past-due-run-test
+ (testing "scheduling a run moves a run pending in the past, which Quartz may not fire for minutes, to a second from now"
+ (mt/with-temp-scheduler!
+ (mt/with-dynamic-fn-redefs [task.util/next-fire-time (constantly (Date. (- (.getTime (Date.)) (* 20 60 1000))))]
+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600))
+ (is (<= 0 (seconds-until-next-run) 1)))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
index a61b623fa59..03119603223 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
@@ -1,21 +1,114 @@
(ns metabase-enterprise.dependencies.test-util
(:require
[medley.core :as m]
- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.result-metadata-status :as models.result-metadata-status]
+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]
+ [metabase-enterprise.dependencies.task.dependency :as task.dependency]
[metabase.app-db.core :as mdb]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util.metadata-providers.mock :as providers.mock]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
-(defn synchronously-run-backfill!
- "Run the dependency backfill job synchronously, processing all stale/outdated entities
- until there is nothing left to process. Use this in tests after operations that mark
- entities stale (create/update events) and before asserting on dependency data."
+(defn run-with-dependencies-setup!
+ "Run `thunk` with the metadata provider of the test database, as Rasta and with the dependencies feature, each write
+ committing on its own as it does between real requests."
+ [thunk]
+ (mt/test-helpers-set-global-values!
+ (mt/with-test-user :rasta
+ (mt/with-premium-features #{:dependencies}
+ (lib-be/with-metadata-provider-cache
+ (let [mp (mt/metadata-provider)]
+ (thunk mp)))))))
+
+(defn synchronously-run-dependency-job!
+ "Run the dependency job synchronously, processing every changed or outdated entity until there is nothing left to
+ process. Use this in tests after writes and before asserting on dependency data."
[]
(mt/with-premium-features #{:dependencies}
- (while (#'dependencies.backfill/backfill-dependencies!))))
+ (while (#'task.dependency/update-dependencies!))))
+
+(defn run-analysis-finding-job!
+ "Run batches of the analysis finding job until one takes nothing, or up to 20 of them, returning whether the last one
+ took anything."
+ []
+ (loop [runs 1]
+ (let [took? (some? (#'task.analysis-finding/run-batch!))]
+ (if (and took? (< runs 20))
+ (recur (inc runs))
+ took?))))
+
+(defn table-query
+ "A query of the test data Table named `table`."
+ [table]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id table)))))
+
+(defn card-query
+ "A query of the Card with `card-id`."
+ [card-id]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/card mp card-id))))
+
+(defn- status-pending?
+ "Whether the entity `entity-type` `entity-id` has no status in the status `model`, or one that counts a write and
+ that the job has not given up on."
+ [model entity-type entity-id]
+ (not (t2/exists? model {:where [:and
+ [:= :entity_type (name entity-type)]
+ [:= :entity_id entity-id]
+ [:or [:= :stale 0] [:= :terminal true]]]})))
+
+(defn record-status-processed!
+ "Record, as a job does when it clears a status, that the entity `entity-type` `entity-id` was processed by the logic
+ of `version`, inserting its status in the status `model` when there is none."
+ [model entity-type entity-id version]
+ (when (zero? (t2/update! model :entity_type entity-type :entity_id entity-id {:stale 0 :analysis_version version}))
+ (t2/insert! model {:entity_type entity-type :entity_id entity-id :stale 0 :analysis_version version})))
+
+(defn dependency-pending?
+ "Whether the dependency job is to compute the dependencies of the entity `entity-type` `entity-id`: it has no
+ status, or one that counts a write."
+ [entity-type entity-id]
+ (status-pending? :model/DependencyStatus entity-type entity-id))
+
+(defn record-dependencies-computed!
+ "Record, as the dependency job does when it clears its status, that the dependencies of the entity `entity-type`
+ `entity-id` are current."
+ [entity-type entity-id]
+ (record-status-processed! :model/DependencyStatus entity-type entity-id
+ models.dependency/current-dependency-analysis-version))
+
+(defn result-metadata-pending?
+ "Whether the result metadata job is to re-derive the Card with `card-id`: it has no status, or one that counts a
+ write."
+ [card-id]
+ (status-pending? :model/ResultMetadataStatus :card card-id))
+
+(defn record-result-metadata-processed!
+ "Record, as the result metadata job does when it clears its status, that the result metadata of the Card with
+ `card-id` is current."
+ [card-id]
+ (record-status-processed! :model/ResultMetadataStatus :card card-id
+ models.result-metadata-status/current-result-metadata-version))
+
+(defn analysis-finding-pending?
+ "Whether the analysis job is to analyze the entity `entity-type` `entity-id`: it has no status, or one that counts a
+ write."
+ [entity-type entity-id]
+ (status-pending? :model/AnalysisFindingStatus entity-type entity-id))
+
+(defn record-analysis-finding-processed!
+ "Record, as the analysis job does when it clears its status, that the finding of the entity `entity-type`
+ `entity-id` is current."
+ [entity-type entity-id]
+ (record-status-processed! :model/AnalysisFindingStatus entity-type entity-id
+ models.analysis-finding/*current-analysis-finding-version*))
(defn do-with-immediate-hook-writes
"Run `thunk` with the work the dependency hooks defer until commit done at once, since a test's transaction never
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
index 6e4c065e2cf..4b7c76473a3 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
@@ -359,7 +359,7 @@
(let [app (t2/select-one :model/DataApp :name "shop")]
(perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
(group-access/add-groups! app [(:id group)])
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]
(group-access/permission-warnings app))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
index f9fe5207c5a..0d3c259eb59 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
@@ -143,7 +143,7 @@
;; Populate dependencies via events
(doseq [card [old-model mbql-child-1 mbql-child-2 native-child grandchild grandchild-native]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
:source_entity_type :card
@@ -173,7 +173,7 @@
(let [src (t2/select-one-fn :source :model/Transform :id transform-id)]
(is (= new-id (lib/primary-source-card-id (:query src))))))
(testing "Dependencies point to new model"
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [edges-to (fn [card-id]
(t2/select-fn-set (juxt :from_entity_type :from_entity_id)
:model/Dependency
@@ -207,7 +207,7 @@
:name "Child card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
;; Insert a fake active run to simulate one already running
(let [run (replacement-run/create-run! :card old-id :card new-id (mt/user->id :crowberto))]
(replacement-run/start-run! (:id run)))
@@ -239,7 +239,7 @@
:name "Child Card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
:source_entity_type :card
@@ -272,7 +272,7 @@
:name "Child card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(snowplow-test/with-fake-snowplow-collector
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
@@ -388,7 +388,7 @@
;; Populate dependencies via events
(doseq [card [model-card question-card]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(snowplow-test/with-fake-snowplow-collector
(let [response (mt/user-http-request :crowberto :post 202
"ee/replacement/replace-model-with-transform"
@@ -509,7 +509,7 @@
(mt/with-model-cleanup [:model/ReplacementRun]
(doseq [card [model-card direct-card nested-card metric-card join-card]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :post 202
"ee/replacement/replace-model-with-transform"
{:card_id model-id
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
index 61704ca4d83..960cdc4ef0a 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
@@ -189,7 +189,7 @@
;; populate dependencies
(events/publish-event! :event/card-create {:object old-card :user-id (mt/user->id :rasta)})
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :rasta)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(testing "child card initially points to old model"
(is (= old-id (get-in (t2/select-one-fn :dataset_query :model/Card :id child-id)
[:stages 0 :source-card]))))
@@ -243,7 +243,7 @@
(mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(doseq [card [old-card child-1 child-2]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :rasta)}))
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(let [original-swap! (mt/original-fn #'replacement.source-swap/swap-source!)]
(mt/with-dynamic-fn-redefs [replacement.source-swap/swap-source!
(fn [entity object old-source new-source]
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
index bdb98152c40..ad757017ed0 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
@@ -419,7 +419,7 @@
;; Create a dependency: card -> orders table
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card card-id)
:user-id (mt/user->id :rasta)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency
:from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :orders))
@@ -449,7 +449,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/segment-create {:object (t2/select-one :model/Segment segment-id)
:user-id (mt/user->id :rasta)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency
:from_entity_type :segment :from_entity_id segment-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
@@ -479,7 +479,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/measure-create {:object (t2/select-one :model/Measure measure-id)
:user-id (mt/user->id :rasta)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency
:from_entity_type :measure :from_entity_id measure-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
@@ -508,7 +508,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/transform-create {:object (t2/select-one :model/Transform transform-id)
:user-id (mt/user->id :rasta)})
- (deps.test/synchronously-run-backfill!)
+ (deps.test/synchronously-run-dependency-job!)
(is (t2/exists? :model/Dependency
:from_entity_type :transform :from_entity_id transform-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
index 487366aa058..b488d7ec8a9 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
@@ -76,7 +76,7 @@
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [source-card (card/create-card! (card-with-query "Source card" :products) user)
child-card (card/create-card! (card-sourced-from "Child card" source-card) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
found-usages (replacement.usages/transitive-usages [:card (:id source-card)])]
(is (some #(= [:card (:id child-card)] %) found-usages)
"Child card should appear in usages"))))))))
@@ -90,7 +90,7 @@
(let [source-card (card/create-card! (card-with-query "Source card" :products) user)
child-card (card/create-card! (card-sourced-from "Child card" source-card) user)
grandchild-card (card/create-card! (card-sourced-from "Grandchild card" child-card) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
found-usages (replacement.usages/transitive-usages [:card (:id source-card)])]
(is (some #(= [:card (:id child-card)] %) found-usages)
"Child card should appear in usages")
@@ -104,7 +104,7 @@
(let [user (mt/fetch-user :rasta)]
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [lonely-card (card/create-card! (card-with-query "Lonely card" :products) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
found-usages (replacement.usages/transitive-usages [:card (:id lonely-card)])]
(is (empty? found-usages)
"Card with no dependents should have empty usages"))))))))
@@ -116,7 +116,7 @@
(let [user (mt/fetch-user :rasta)]
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [card-on-table (card/create-card! (card-with-query "Card on products" :products) user)
- _ (deps.test/synchronously-run-backfill!)
+ _ (deps.test/synchronously-run-dependency-job!)
found-usages (replacement.usages/transitive-usages [:table (mt/id :products)])]
(is (some #(= [:card (:id card-on-table)] %) found-usages)
"Card using table should appear in usages"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 120e6e3f686..6be8f4f449d 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -37,6 +37,7 @@
:model/AiUsageLog
:model/AnalysisFinding
:model/AnalysisFindingError
+ :model/AnalysisFindingStatus
:model/ApiKey
:model/ApiKeyUsageLog
:model/AuthIdentity
@@ -122,6 +123,7 @@
:model/ReplacementRun
:model/RemoteSyncObject
:model/RemoteSyncTask
+ :model/ResultMetadataStatus
:model/Revision
:model/SemanticSearchTokenTracking
:model/SearchIndexMetadata
diff --git a/resources/migrations/065/20261018_dependency_change_detection.yaml b/resources/migrations/065/20261018_dependency_change_detection.yaml
new file mode 100644
index 00000000000..b7cbbd6f7cb
--- /dev/null
+++ b/resources/migrations/065/20261018_dependency_change_detection.yaml
@@ -0,0 +1,503 @@
+## The dependency jobs process the entities whose status rows a write marks stale in its own transaction: an
+## entity's own row for the columns each job reads, and its direct dependents' rows when what they read changes. The
+## status rows lose the retry delays and keep a failure count and a terminal flag; their stale flag becomes a count
+## of the writes the job has not read. result_metadata_status and analysis_finding_status hold one row per entity,
+## as dependency_status does, and analysis_finding keeps only the findings.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.drkztr
+ author: ranquild
+ comment: Drop dependency_status.next_retry_at, which the jobs no longer use since they don't retry
+ changes:
+ - dropColumn:
+ tableName: dependency_status
+ columnName: next_retry_at
+ rollback:
+ - addColumn:
+ tableName: dependency_status
+ columns:
+ - column:
+ name: next_retry_at
+ type: ${timestamp_type}
+ remarks: Earliest time at which a failed entity can be retried (null means immediately processable)
+
+ - changeSet:
+ id: v65.u3r8zm
+ author: ranquild
+ comment: Drop dependency_status.unresolved_tables, since a table a native query names is recorded once the query changes again
+ changes:
+ - dropColumn:
+ tableName: dependency_status
+ columnName: unresolved_tables
+ rollback:
+ - addColumn:
+ tableName: dependency_status
+ columns:
+ - column:
+ name: unresolved_tables
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity names tables that match no synced table, recomputed after each sync
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v65.lywci7
+ author: ranquild
+ comment: Rename dependency_status.dependency_analysis_version to analysis_version, as the other status tables name it
+ changes:
+ - renameColumn:
+ tableName: dependency_status
+ oldColumnName: dependency_analysis_version
+ newColumnName: analysis_version
+ columnDataType: smallint
+ rollback:
+ - renameColumn:
+ tableName: dependency_status
+ oldColumnName: analysis_version
+ newColumnName: dependency_analysis_version
+ columnDataType: smallint
+ - addNotNullConstraint:
+ tableName: dependency_status
+ columnName: dependency_analysis_version
+ columnDataType: smallint
+ - addDefaultValue:
+ tableName: dependency_status
+ columnName: dependency_analysis_version
+ columnDataType: smallint
+ defaultValueNumeric: 0
+
+ - changeSet:
+ id: v65.0mshmu
+ author: ranquild
+ comment: Keep dependency_status.analysis_version not null, which a rename on MySQL drops
+ changes:
+ - addNotNullConstraint:
+ tableName: dependency_status
+ columnName: analysis_version
+ columnDataType: smallint
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.wlb1z1
+ author: ranquild
+ comment: Keep the default of dependency_status.analysis_version, which a rename on MySQL drops
+ changes:
+ - addDefaultValue:
+ tableName: dependency_status
+ columnName: analysis_version
+ columnDataType: smallint
+ defaultValueNumeric: 0
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.oacw7z
+ author: ranquild
+ comment: Create result_metadata_status
+ changes:
+ - createTable:
+ tableName: result_metadata_status
+ remarks: Whether the result metadata of each MBQL card is to be re-derived
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: entity_type
+ type: varchar(254)
+ remarks: The type of the entity
+ constraints:
+ nullable: false
+ - column:
+ name: entity_id
+ type: int
+ remarks: The ID of the entity
+ constraints:
+ nullable: false
+ - column:
+ name: stale
+ type: int
+ defaultValueNumeric: 0
+ remarks: How many writes changed what the job reads for the entity since the job last read it; the job processes it while above 0
+ constraints:
+ nullable: false
+ - column:
+ name: fail_count
+ type: int
+ defaultValueNumeric: 0
+ remarks: How many times in a row the job failed to process the entity since a write changed it
+ constraints:
+ nullable: false
+ - column:
+ name: terminal
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the job gave up on the entity after its failures, until a write changes it
+ constraints:
+ nullable: false
+ - column:
+ name: analysis_version
+ type: int
+ remarks: The version of the result metadata logic that last processed the card
+ constraints:
+ nullable: false
+ rollback:
+ - dropTable:
+ tableName: result_metadata_status
+
+ - changeSet:
+ id: v65.ckqf9u
+ author: ranquild
+ comment: Unique constraint to prevent duplicate result_metadata_status entries
+ changes:
+ - addUniqueConstraint:
+ tableName: result_metadata_status
+ columnNames: entity_type, entity_id
+ constraintName: idx_unique_result_metadata_status
+ rollback:
+ - dropUniqueConstraint:
+ tableName: result_metadata_status
+ constraintName: idx_unique_result_metadata_status
+
+ - changeSet:
+ id: v65.u44qob
+ author: ranquild
+ comment: Create analysis_finding_status
+ changes:
+ - createTable:
+ tableName: analysis_finding_status
+ remarks: Whether each analyzable entity is to be analyzed again
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: entity_type
+ type: varchar(254)
+ remarks: The type of the entity
+ constraints:
+ nullable: false
+ - column:
+ name: entity_id
+ type: int
+ remarks: The ID of the entity
+ constraints:
+ nullable: false
+ - column:
+ name: stale
+ type: int
+ defaultValueNumeric: 0
+ remarks: How many writes changed what the job reads for the entity since the job last read it; the job processes it while above 0
+ constraints:
+ nullable: false
+ - column:
+ name: fail_count
+ type: int
+ defaultValueNumeric: 0
+ remarks: How many times in a row the job failed to process the entity since a write changed it
+ constraints:
+ nullable: false
+ - column:
+ name: terminal
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the job gave up on the entity after its failures, until a write changes it
+ constraints:
+ nullable: false
+ - column:
+ name: analysis_version
+ type: int
+ remarks: The version of the analysis that last processed the entity
+ constraints:
+ nullable: false
+ rollback:
+ - dropTable:
+ tableName: analysis_finding_status
+
+ - changeSet:
+ id: v65.0ypnl2
+ author: ranquild
+ comment: Unique constraint to prevent duplicate analysis_finding_status entries
+ changes:
+ - addUniqueConstraint:
+ tableName: analysis_finding_status
+ columnNames: entity_type, entity_id
+ constraintName: idx_unique_analysis_finding_status
+ rollback:
+ - dropUniqueConstraint:
+ tableName: analysis_finding_status
+ constraintName: idx_unique_analysis_finding_status
+
+ - changeSet:
+ id: v65.eoug8w
+ author: ranquild
+ comment: Copy the stale flag and analysis version of each analysis_finding row to analysis_finding_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO analysis_finding_status (entity_type, entity_id, stale, analysis_version)
+ SELECT analyzed_entity_type, analyzed_entity_id, CASE WHEN stale THEN 1 ELSE 0 END, COALESCE(analysis_version, 0)
+ FROM analysis_finding
+ rollback: # the table is dropped by the rollback of the changeset creating it
+
+ - changeSet:
+ id: v65.1suf0c
+ author: ranquild
+ comment: Drop the index on analysis_finding.stale, which moves to analysis_finding_status
+ changes:
+ - dropIndex:
+ tableName: analysis_finding
+ indexName: idx_analysis_finding_stale
+ rollback:
+ - sql:
+ dbms: postgresql
+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale) WHERE stale = true
+ - sql:
+ dbms: mysql,mariadb,h2
+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale)
+
+ - changeSet:
+ id: v65.d291qd
+ author: ranquild
+ comment: Drop analysis_finding.stale, which analysis_finding_status now holds
+ changes:
+ - dropColumn:
+ tableName: analysis_finding
+ columnName: stale
+ rollback:
+ - addColumn:
+ tableName: analysis_finding
+ columns:
+ - column:
+ name: stale
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity needs re-analysis due to upstream changes
+ constraints:
+ nullable: false
+ - sql:
+ sql: >-
+ UPDATE analysis_finding SET stale = COALESCE((SELECT s.stale > 0
+ FROM analysis_finding_status s WHERE s.entity_type = analysis_finding.analyzed_entity_type
+ AND s.entity_id = analysis_finding.analyzed_entity_id), FALSE)
+
+ - changeSet:
+ id: v65.em1iri
+ author: ranquild
+ comment: Drop analysis_finding.analysis_version, which analysis_finding_status records
+ changes:
+ - dropColumn:
+ tableName: analysis_finding
+ columnName: analysis_version
+ rollback:
+ - addColumn:
+ tableName: analysis_finding
+ columns:
+ - column:
+ name: analysis_version
+ type: int
+ defaultValueNumeric: 0
+ remarks: The version of the analysis that was performed
+ constraints:
+ nullable: false
+ - sql:
+ sql: >-
+ UPDATE analysis_finding SET analysis_version = COALESCE((SELECT s.analysis_version
+ FROM analysis_finding_status s WHERE s.entity_type = analysis_finding.analyzed_entity_type
+ AND s.entity_id = analysis_finding.analyzed_entity_id), 0)
+
+ - changeSet:
+ id: v65.pxgfgg
+ author: ranquild
+ comment: Rename analysis_finding.analyzed_entity_type to entity_type, as the status tables name it
+ changes:
+ - renameColumn:
+ tableName: analysis_finding
+ oldColumnName: analyzed_entity_type
+ newColumnName: entity_type
+ columnDataType: varchar(254)
+ rollback:
+ - renameColumn:
+ tableName: analysis_finding
+ oldColumnName: entity_type
+ newColumnName: analyzed_entity_type
+ columnDataType: varchar(254)
+ - addNotNullConstraint:
+ tableName: analysis_finding
+ columnName: analyzed_entity_type
+ columnDataType: varchar(254)
+
+ - changeSet:
+ id: v65.2rxyyj
+ author: ranquild
+ comment: Rename analysis_finding.analyzed_entity_id to entity_id, as the status tables name it
+ changes:
+ - renameColumn:
+ tableName: analysis_finding
+ oldColumnName: analyzed_entity_id
+ newColumnName: entity_id
+ columnDataType: int
+ rollback:
+ - renameColumn:
+ tableName: analysis_finding
+ oldColumnName: entity_id
+ newColumnName: analyzed_entity_id
+ columnDataType: int
+ - addNotNullConstraint:
+ tableName: analysis_finding
+ columnName: analyzed_entity_id
+ columnDataType: int
+
+ - changeSet:
+ id: v65.p5wigm
+ author: ranquild
+ comment: Keep analysis_finding.entity_type not null, which a rename on MySQL drops
+ changes:
+ - addNotNullConstraint:
+ tableName: analysis_finding
+ columnName: entity_type
+ columnDataType: varchar(254)
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.j3y4t5
+ author: ranquild
+ comment: Keep analysis_finding.entity_id not null, which a rename on MySQL drops
+ changes:
+ - addNotNullConstraint:
+ tableName: analysis_finding
+ columnName: entity_id
+ columnDataType: int
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.7d5dj3
+ author: ranquild
+ comment: Drop the index on dependency_status.stale, which becomes a count of the writes the job has not read
+ changes:
+ - dropIndex:
+ tableName: dependency_status
+ indexName: idx_dependency_status_stale
+ rollback:
+ - sql:
+ dbms: postgresql
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale = true
+ - sql:
+ dbms: mysql,mariadb,h2
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)
+
+ - changeSet:
+ id: v65.47bljg
+ author: ranquild
+ comment: Add dependency_status.stale_count, which replaces the stale flag
+ changes:
+ - addColumn:
+ tableName: dependency_status
+ columns:
+ - column:
+ name: stale_count
+ type: int
+ defaultValueNumeric: 0
+ remarks: How many writes changed what the job reads for the entity since the job last read it
+ constraints:
+ nullable: false
+ rollback:
+ - dropColumn:
+ tableName: dependency_status
+ columnName: stale_count
+
+ - changeSet:
+ id: v65.i33b3h
+ author: ranquild
+ comment: Copy the dependency_status stale flags into stale_count
+ changes:
+ - sql:
+ sql: UPDATE dependency_status SET stale_count = CASE WHEN stale THEN 1 ELSE 0 END
+ rollback:
+ - sql:
+ sql: UPDATE dependency_status SET stale = (stale_count > 0)
+
+ - changeSet:
+ id: v65.vcn86w
+ author: ranquild
+ comment: Drop the dependency_status stale flag, which stale_count replaces
+ changes:
+ - dropColumn:
+ tableName: dependency_status
+ columnName: stale
+ rollback:
+ - addColumn:
+ tableName: dependency_status
+ columns:
+ - column:
+ name: stale
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity needs (re)processing
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v65.accafc
+ author: ranquild
+ comment: Rename dependency_status.stale_count to stale, as the other status tables name it
+ changes:
+ - renameColumn:
+ tableName: dependency_status
+ oldColumnName: stale_count
+ newColumnName: stale
+ columnDataType: int
+ rollback:
+ - renameColumn:
+ tableName: dependency_status
+ oldColumnName: stale
+ newColumnName: stale_count
+ columnDataType: int
+
+ - changeSet:
+ id: v65.cw6b2z
+ author: ranquild
+ comment: Keep dependency_status.stale not null, which a rename on MySQL drops
+ changes:
+ - addNotNullConstraint:
+ tableName: dependency_status
+ columnName: stale
+ columnDataType: int
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.hn74zg
+ author: ranquild
+ comment: Keep the default of dependency_status.stale, which a rename on MySQL drops
+ changes:
+ - addDefaultValue:
+ tableName: dependency_status
+ columnName: stale
+ columnDataType: int
+ defaultValueNumeric: 0
+ rollback: # the rename's rollback restores the column
+
+ - changeSet:
+ id: v65.f2ttgl
+ author: ranquild
+ comment: Index dependency_status.stale, the count of writes the job has not read
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale > 0
+ - sql:
+ dbms: mysql,mariadb,h2
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)
+ rollback:
+ - dropIndex:
+ tableName: dependency_status
+ indexName: idx_dependency_status_stale
diff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.clj
index 17a48fed6e4..89cfa4a0d34 100644
--- a/src/metabase/batch_processing/impl.clj
+++ b/src/metabase/batch_processing/impl.clj
@@ -110,7 +110,7 @@
sdk-tracked-row
id+timestamp
[:map {:closed true}
- [:table-id ms/PositiveInt]]
+ [:dependency-jobs [:= true]]]
[:map {:closed true}
[:host :string]
[:timestamp (ms/InstanceOfClass java.time.temporal.Temporal)]]
diff --git a/src/metabase/dependencies/hooks.clj b/src/metabase/dependencies/hooks.clj
index c7504438bd7..1d863c42911 100644
--- a/src/metabase/dependencies/hooks.clj
+++ b/src/metabase/dependencies/hooks.clj
@@ -1,50 +1,48 @@
(ns metabase.dependencies.hooks
- "Hooks that keep the dependency graph current as content is written, implemented by the enterprise module."
+ "Hooks that have the dependency jobs triggered as content is written, implemented by the enterprise module."
(:require
[metabase.premium-features.defenterprise :refer [defenterprise]]))
(defenterprise before-insert
- "Hook for the `model` row `instance` before it is inserted. The OSS implementation is a no-op."
+ "Hook for `row` of `model` before it is inserted, which does nothing without the dependencies feature."
metabase-enterprise.dependencies.hooks
- [_model _instance]
+ [_model _row]
nil)
(defenterprise before-update
- "Hook for the `model` row `instance` before it is updated, whose Toucan changes say what changes. The OSS
- implementation is a no-op."
+ "Hook for `row` of `model` before it is updated, which does nothing without the dependencies feature."
metabase-enterprise.dependencies.hooks
- [_model _instance]
+ [_model _row]
nil)
(defenterprise before-delete
- "Hook for the `model` row `instance` before it is deleted. The OSS implementation is a no-op."
+ "Hook for `row` of `model` before it is deleted, which does nothing without the dependencies feature."
metabase-enterprise.dependencies.hooks
- [_model _instance]
- nil)
-
-(defenterprise after-database-sync
- "Hook for the Database with `database-id` after a sync. The OSS implementation is a no-op."
- metabase-enterprise.dependencies.hooks
- [_database-id]
- nil)
-
-(defenterprise after-serdes-load
- "Hook for after a serdes load, during which the hooks mark what the load changes without triggering the jobs. The
- OSS implementation is a no-op."
- metabase-enterprise.dependencies.hooks
- []
+ [_model _row]
nil)
(defenterprise after-premium-token-update
- "Hook for after the premium token changed, which may have enabled the dependencies feature. The OSS implementation is
- a no-op."
+ "Hook for after the premium token changed, which may have enabled the dependencies feature, and does nothing without
+ it."
metabase-enterprise.dependencies.hooks
[]
nil)
(defenterprise after-transform-run
"Hook for the transform with `transform-id` after a run wrote its output Table with `table-id`, or nil when the
- table was not synced. The OSS implementation is a no-op."
+ table was not synced, which does nothing without the dependencies feature."
metabase-enterprise.dependencies.hooks
[_transform-id _table-id]
nil)
+
+(defenterprise do-with-batched-marks
+ "Run `thunk`, with the dependents of what its writes change marked once each when it ends."
+ metabase-enterprise.dependencies.hooks
+ [thunk]
+ (thunk))
+
+(defmacro with-batched-marks
+ "Run `body`, with the dependents of what its writes change marked once each when it ends, for a batch of writes
+ such as a sync's."
+ [& body]
+ `(do-with-batched-marks (fn [] ~@body)))
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 0e6edf8dd3d..2cbc81bfff1 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -1612,7 +1612,7 @@
to get the raw native query as needed.
A table the query names that matches no synced table or transform target can be returned as
- `{:unresolved-table {:schema ... :table ...}}`, so dependency tracking can recompute the query once sync finds it.
+ `{:unresolved-table {:schema ... :table ...}}`, which dependency tracking does not record as an upstream.
The return value should match the `:metabase.driver/native-query-deps` schema."
{:added "0.57.0" :arglists '([driver query])}
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index dde7789d5ed..e3a5ed1a078 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -1582,6 +1582,7 @@
metadata-provider?
metadata-providerable?]
[metabase.lib.metadata.result-metadata
+ match-returned-columns
normalize-result-metadata-column]
[lib.native
add-parameters-for-template-tags
diff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljc
index d782800ec7e..c01dbba6a88 100644
--- a/src/metabase/lib/metadata/result_metadata.cljc
+++ b/src/metabase/lib/metadata/result_metadata.cljc
@@ -557,3 +557,16 @@
(->> col
(lib.normalize/normalize :metabase.query-processor.schema/result-metadata.column)))
:cljs (lib.normalize/normalize ::lib.schema.metadata/column col)))
+
+(defn match-returned-columns
+ "For each of the newly returned columns `cols`, the returned column of `stored-cols` it stands for, or nil: the one
+ with the same desired column alias, or else the one with the same name when either has no alias."
+ [stored-cols cols]
+ (let [by-alias (m/index-by :lib/desired-column-alias (filter :lib/desired-column-alias stored-cols))
+ by-name (m/index-by :name stored-cols)]
+ (mapv (fn [col]
+ (or (get by-alias (:lib/desired-column-alias col))
+ (let [stored-col (get by-name (:name col))]
+ (when (not-every? :lib/desired-column-alias [col stored-col])
+ stored-col))))
+ cols)))
diff --git a/src/metabase/lib/schema/validate.cljc b/src/metabase/lib/schema/validate.cljc
index f6569de8ad1..d9060478b7a 100644
--- a/src/metabase/lib/schema/validate.cljc
+++ b/src/metabase/lib/schema/validate.cljc
@@ -4,39 +4,39 @@
[metabase.util.malli.registry :as mr]))
(mr/def ::missing-column-error
- [:map
- [:type [:= :missing-column]]
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-column]]
[:name :string]])
(mr/def ::missing-table-alias-error
- [:map
- [:type [:= :missing-table-alias]]
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-table-alias]]
[:name :string]])
(mr/def ::missing-table-error
"A stage's `:source-table` references a table that is missing or inactive (e.g. dropped during sync).
The offending table is identified by the `:source-entity-*` keys, so no `:name` is carried."
- [:map
- [:type [:= :missing-table]]])
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-table]]])
(mr/def ::missing-card-error
"A stage's `:source-card` references a card that has been deleted.
The offending card is identified by the `:source-entity-*` keys, so no `:name` is carried."
- [:map
- [:type [:= :missing-card]]])
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-card]]])
(mr/def ::duplicate-column-error
- [:map
- [:type [:= :duplicate-column]]
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :duplicate-column]]
[:name :string]])
(mr/def ::syntax-error
- [:map
- [:type [:= :syntax-error]]])
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :syntax-error]]])
(mr/def ::validation-exception-error
- [:map
- [:type [:= :validation-exception-error]]
+ [:map {:closed true}
+ [:type {:decode/normalize common/normalize-keyword} [:= :validation-exception-error]]
[:message :string]])
(mr/def ::validate-error-type
@@ -44,16 +44,14 @@
:validation-exception-error])
(mr/def ::error
- [:and
- [:map [:type {:decode/normalize common/normalize-keyword} ::validate-error-type]]
- [:multi {:dispatch #(-> % :type keyword)}
- [:missing-column [:ref ::missing-column-error]]
- [:missing-table-alias [:ref ::missing-table-alias-error]]
- [:missing-table [:ref ::missing-table-error]]
- [:missing-card [:ref ::missing-card-error]]
- [:duplicate-column [:ref ::duplicate-column-error]]
- [:syntax-error [:ref ::syntax-error]]
- [:validation-exception-error [:ref ::validation-exception-error]]]])
+ [:multi {:dispatch #(some-> % :type keyword)}
+ [:missing-column [:ref ::missing-column-error]]
+ [:missing-table-alias [:ref ::missing-table-alias-error]]
+ [:missing-table [:ref ::missing-table-error]]
+ [:missing-card [:ref ::missing-card-error]]
+ [:duplicate-column [:ref ::duplicate-column-error]]
+ [:syntax-error [:ref ::syntax-error]]
+ [:validation-exception-error [:ref ::validation-exception-error]]])
(mr/def ::source-entity-type
"The type of the source entity causing an error."
@@ -61,23 +59,23 @@
(mr/def ::source-entity
"Optional source entity tracking fields for dependency analysis."
- [:map
+ [:map {:closed true}
[:source-entity-type {:optional true} [:maybe ::source-entity-type]]
[:source-entity-id {:optional true} [:maybe pos-int?]]])
+(mr/def ::soft
+ "Whether the offending ref could be dropped, e.g. from a `:fields` list, without breaking the query."
+ [:map {:closed true}
+ [:soft? {:optional true} :boolean]])
+
(mr/def ::error-with-source
"An error with optional source entity information.
Extends the base error types with source tracking for dependency analysis."
- [:and
- [:map
- [:type {:decode/normalize common/normalize-keyword} ::validate-error-type]
- [:source-entity-type {:optional true} [:maybe ::source-entity-type]]
- [:source-entity-id {:optional true} [:maybe pos-int?]]]
- [:multi {:dispatch #(-> % :type keyword)}
- [:missing-column [:merge [:ref ::missing-column-error] [:ref ::source-entity]]]
- [:missing-table-alias [:merge [:ref ::missing-table-alias-error] [:ref ::source-entity]]]
- [:missing-table [:merge [:ref ::missing-table-error] [:ref ::source-entity]]]
- [:missing-card [:merge [:ref ::missing-card-error] [:ref ::source-entity]]]
- [:duplicate-column [:merge [:ref ::duplicate-column-error] [:ref ::source-entity]]]
- [:syntax-error [:merge [:ref ::syntax-error] [:ref ::source-entity]]]
- [:validation-exception-error [:merge [:ref ::validation-exception-error] [:ref ::source-entity]]]]])
+ [:multi {:dispatch #(some-> % :type keyword)}
+ [:missing-column [:merge [:ref ::missing-column-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:missing-table-alias [:merge [:ref ::missing-table-alias-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:missing-table [:merge [:ref ::missing-table-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:missing-card [:merge [:ref ::missing-card-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:duplicate-column [:merge [:ref ::duplicate-column-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:syntax-error [:merge [:ref ::syntax-error] [:ref ::source-entity] [:ref ::soft]]]
+ [:validation-exception-error [:merge [:ref ::validation-exception-error] [:ref ::source-entity] [:ref ::soft]]]])
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 8c56d7a6f94..9ec9ac31f52 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -694,7 +694,7 @@
(methodical/defmethod t2.pipeline/transduce-query :around [:toucan.query-type/update.* :hook/dependencies :default]
"Run the before-update hooks of a `:hook/dependencies` model in the transaction of its UPDATE, which Toucan otherwise
- opens only after them, so dependency marks commit together with the UPDATE and after-commit work sees it."
+ opens only after them, so the dependency marks commit or roll back with the UPDATE and after-commit work sees it."
[rf query-type model parsed-args resolved-query]
(t2.conn/with-transaction [_conn nil {:nested-transaction-rule :ignore}]
(next-method rf query-type model parsed-args resolved-query)))
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 4bafbc3fc6b..13f231a1b50 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -16,6 +16,7 @@
:model/AgentApiCallLog metabase.agent-api.models.agent-api-call-log
:model/AnalysisFinding metabase-enterprise.dependencies.models.analysis-finding
:model/AnalysisFindingError metabase-enterprise.dependencies.models.analysis-finding-error
+ :model/AnalysisFindingStatus metabase-enterprise.dependencies.models.analysis-finding-status
:model/ApiKey metabase.api-keys.models.api-key
:model/ApiKeyUsageLog metabase.api-keys.models.api-key-usage-log
:model/ApplicationPermissionsRevision metabase.permissions.models.application-permissions-revision
@@ -126,6 +127,7 @@
:model/RemoteSyncObject metabase-enterprise.remote-sync.models.remote-sync-object
:model/RemoteSyncTask metabase-enterprise.remote-sync.models.remote-sync-task
:model/ReplacementRun metabase-enterprise.replacement.models.replacement-run
+ :model/ResultMetadataStatus metabase-enterprise.dependencies.models.result-metadata-status
:model/Revision metabase.revisions.models.revision
:model/SearchIndexMetadata metabase.search.models.search-index-metadata
:model/Secret metabase.secrets.models.secret
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 2ee048f1ea7..3c4b71891f4 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1273,10 +1273,7 @@
;; coerced to `{}`. See #74615.
(m/update-existing :dataset_query #(lib-be/normalize-query nil % {:strict? true})))
{:keys [metadata metadata-future]} (card.metadata/maybe-async-result-metadata
- ;; 1. This function is called when storing metadata.
- ;; 2. The metadata for storage shouldn't have remaps.
- ;; 3. Setting this keyword will keep the remapped fields out of the metadata.
- {:query (assoc-in (:dataset_query card-data) [:middleware :disable-remaps?] true)
+ {:query (:dataset_query card-data)
:metadata result_metadata
:entity-id (:entity_id card-data)
:model? (model? card-data)})
@@ -1512,13 +1509,11 @@
:result_metadata :collection_preview :verified-result-metadata?})]
(assert-is-valid-dashboard-internal-update card-updates card-before-update)
(maybe-unverify! card-before-update updated-fields actor)
- ;; Invalidate the cache for card
- (cache/invalidate-config! {:questions [(:id card-before-update)]
- :with-overrides? true})
- ;; ok, now save the Card
(queries.db/update-card! (:id card-before-update)
(m/update-existing updated-fields :dataset_query
- #(lib/normalize ::queries.schema/card.dataset-query %))))
+ #(lib/normalize ::queries.schema/card.dataset-query %)))
+ (cache/invalidate-config! {:questions [(:id card-before-update)]
+ :with-overrides? true}))
;; Update all transitively dependent cards if the database was changed (#74561)
(cascade-database-change-to-dependents! card-before-update card-updates)
;; ok, now update dependent dashcard parameters
diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.clj
index 3f4ecf26fb4..ee3efc02ac7 100644
--- a/src/metabase/queries/models/card/metadata.clj
+++ b/src/metabase/queries/models/card/metadata.clj
@@ -187,17 +187,11 @@ saved later when it is ready."
(log/errorf "Error updating metadata for Card %d asynchronously: %s" id (ex-message e)))))))
(defn infer-metadata
- "Infer the default result_metadata to store for MBQL cards.
-
- Ignores any that might be present already."
+ "Infer the default result_metadata to store for MBQL cards, as no user, ignoring any already present."
[query]
(not-empty (request/with-current-user nil
(u/ignore-exceptions
- (qp.preprocess/query->expected-cols
- ;; 1. This function is called when storing metadata.
- ;; 2. The metadata for storage shouldn't have remaps.
- ;; 3. Setting this keyword will keep the remapped fields out of the metadata.
- (assoc-in query [:middleware :disable-remaps?] true))))))
+ (qp.preprocess/query->expected-cols query)))))
(defn infer-metadata-with-model-overrides
"Does a fresh [[infer-metadata]] for the provided query.
diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.clj
index e0a8dc37322..425b2b9ae96 100644
--- a/src/metabase/query_processor/middleware/add_remaps.clj
+++ b/src/metabase/query_processor/middleware/add_remaps.clj
@@ -639,6 +639,11 @@
((map remap-fn) rf)
rf))
+(defn- cols-internal-columns-info
+ "The [[internal-columns-info]] of the result columns `cols`."
+ [cols]
+ (internal-columns-info (map #(lib-be/instance->metadata % :metadata/column) cols)))
+
(mu/defn remap-results :- ::qp.schema/rff
"Post-processing middleware. Handles `::external-remaps` added by [[add-remapped-columns-middleware]]; transforms
results and adds additional metadata based on these remaps, as well as internal (human-readable values) remaps."
@@ -647,13 +652,19 @@
(if disable-remaps?
rff
(fn remap-results-rff* [metadata]
- (let [lib-cols (map
- #(lib-be/instance->metadata % :metadata/column)
- (:cols metadata))
- internal-cols-info (internal-columns-info lib-cols)
+ (let [internal-cols-info (cols-internal-columns-info (:cols metadata))
metadata (add-remapped-to-and-from-metadata metadata external-remaps internal-cols-info)]
(remap-results-xform internal-cols-info (rff metadata))))))
+(mu/defn remap-cols :- [:maybe ::qp.schema/result-metadata.columns]
+ "The result columns `cols` of the preprocessed `query` with the remapping metadata [[remap-results]] adds to the
+ results of running it."
+ [{::keys [external-remaps], {:keys [disable-remaps?]} :middleware, :as _query} :- ::lib.schema/query
+ cols :- [:maybe ::qp.schema/result-metadata.columns]]
+ (if (or disable-remaps? (empty? cols))
+ cols
+ (vec (:cols (add-remapped-to-and-from-metadata {:cols cols} external-remaps (cols-internal-columns-info cols))))))
+
(defn disable-remaps
"Sets the value of the disable-remaps? option in this query."
[query]
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index c62b87566bd..1a96747dbb5 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -164,4 +164,4 @@
(let [preprocessed (-> query preprocess)]
;; TODO - we should throw an Exception if the query has a native source query with no attached metadata or at
;; least warn about it. Need to check where this is used.
- (not-empty (annotate/expected-cols preprocessed)))))
+ (not-empty (qp.add-remaps/remap-cols preprocessed (annotate/expected-cols preprocessed))))))
diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.clj
index 0435c95b92d..ce1ed024adb 100644
--- a/src/metabase/query_processor/util.clj
+++ b/src/metabase/query_processor/util.clj
@@ -4,7 +4,6 @@
(:require
[buddy.core.codecs :as codecs]
[clojure.string :as str]
- [medley.core :as m]
[metabase.driver :as driver]
[metabase.lib-be.core]
[metabase.lib.core :as lib]
@@ -92,17 +91,11 @@
[new-metadata old-metadata]
;; deprecated fn using its own deprecated helper; both leave together
#_{:clj-kondo/ignore [:deprecated-var]}
- (let [old-cols-by-desired-col-alias (m/index-by :lib/desired-column-alias (filter :lib/desired-column-alias old-metadata))
- old-cols-by-name (m/index-by :name old-metadata)]
- (for [new-col new-metadata]
- (if-let [old-col (or (get old-cols-by-desired-col-alias (:lib/desired-column-alias new-col))
- ;; Only match by name if a desired column alias is missing,
- ;; otherwise we could match different columns with the same name
- (let [old-col (get old-cols-by-name (:name new-col))]
- (when (not-every? :lib/desired-column-alias [new-col old-col])
- old-col)))]
- (merge new-col (select-keys old-col preserved-keys))
- new-col))))
+ (map (fn [new-col old-col]
+ (cond-> new-col
+ old-col (merge (select-keys old-col preserved-keys))))
+ new-metadata
+ (lib/match-returned-columns old-metadata new-metadata)))
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *execute-async?*
diff --git a/src/metabase/sync/analyze/classify.clj b/src/metabase/sync/analyze/classify.clj
index 93d1408d07f..f0f1415f593 100644
--- a/src/metabase/sync/analyze/classify.clj
+++ b/src/metabase/sync/analyze/classify.clj
@@ -19,6 +19,7 @@
(:require
[clojure.data :as data]
[metabase.analyze.core :as analyze]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]
[metabase.models.interface :as mi]
@@ -118,16 +119,17 @@
(when-let [fields (fields-to-classify table)]
(let [existing-name-field (sync.db/name-field-count-for-table table-id)
{:keys [fields-failed]}
- (reduce (fn [state field]
- (let [result (classify! field state)]
- (cond-> state
- (instance? Exception result)
- (update :fields-failed inc)
-
- (= :type/Name (:semantic_type result))
- (assoc :exists-name true))))
- {:fields-failed 0 :exists-name (pos? existing-name-field)}
- fields)]
+ (dependencies.hooks/with-batched-marks
+ (reduce (fn [state field]
+ (let [result (classify! field state)]
+ (cond-> state
+ (instance? Exception result)
+ (update :fields-failed inc)
+
+ (= :type/Name (:semantic_type result))
+ (assoc :exists-name true))))
+ {:fields-failed 0 :exists-name (pos? existing-name-field)}
+ fields))]
{:fields-classified (count fields)
:fields-failed fields-failed})))))
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index f64a7a0b989..f1aa269e146 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -7,7 +7,6 @@
`toucan2.core` itself."
(:require
[clojure.set :as set]
- [honey.sql :as sql]
[metabase.app-db.core :as app-db]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util :as u]
@@ -562,53 +561,6 @@
[:= :t.active true]
[:= :t.visibility_type nil]]})
-(defn- fk-target-changed-clause
- "Honey SQL clause true when the Field's `fk_target_field_id` is not already `pk-id-expr`."
- [pk-id-expr]
- [:or
- [:= :f.fk_target_field_id nil]
- [:not= :f.fk_target_field_id pk-id-expr]])
-
-(defn- mark-fk-statement
- "`[sql & params]` updating the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name
- fk-column-name]` in the Database with `db-id` to the id of the Field at `[pk-table-schema pk-table-name
- pk-column-name]`, unless it already points there, per the application DB's dialect."
- [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]
- (let [fk-field-id-query (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)
- pk-field-id-query (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)
- q (case (app-db/db-type)
- :mysql
- {:update [(t2/table-name :model/Field) :f]
- :join [[fk-field-id-query :fk] [:= :fk.id :f.id]
- [pk-field-id-query :pk]
- (fk-target-changed-clause :pk.id)]
- :set {:fk_target_field_id :pk.id
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}}
- :postgres
- {:update [(t2/table-name :model/Field) :f]
- :from [[fk-field-id-query :fk]]
- :join [[pk-field-id-query :pk] true]
- :set {:fk_target_field_id :pk.id
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}
- :where [:and
- [:= :fk.id :f.id]
- (fk-target-changed-clause :pk.id)]}
- :h2
- {:update [(t2/table-name :model/Field) :f]
- :set {:fk_target_field_id pk-field-id-query
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}
- :where [:and
- [:= :f.id fk-field-id-query]
- [:not= pk-field-id-query nil]
- (fk-target-changed-clause pk-field-id-query)]})]
- (sql/format q :dialect (app-db/quoting-style (app-db/db-type)))))
-
(mu/defn mark-fk!
"Set the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name fk-column-name]` in the Database with
`db-id` to the id of the Field at `[pk-table-schema pk-table-name pk-column-name]`, unless it already points there.
@@ -620,10 +572,15 @@
pk-table-schema :- [:maybe :string]
pk-table-name :- :string
pk-column-name :- :string]
- (t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-name
- pk-table-schema pk-table-name pk-column-name)))
-
-;;; ------------------------------------------ Field data sensitivity ------------------------------------------
+ (let [fk-id (:id (t2/query-one (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)))
+ pk-id (:id (t2/query-one (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)))
+ field (when (and fk-id pk-id)
+ (t2/select-one [:model/Field :id :fk_target_field_id :has_field_values] :id fk-id
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))]
+ (if (and field (not= pk-id (:fk_target_field_id field)))
+ (t2/update! :model/Field fk-id (cond-> {:fk_target_field_id pk-id :semantic_type :type/FK}
+ (= :auto-list (:has_field_values field)) (assoc :has_field_values nil)))
+ 0)))
(defn- data-sensitivity-to-scan-clause
"Honey SQL clause matching Fields the data-sensitivity classifier still has to scan: unlabeled ones, plus those it
diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.clj
index 49b4f4cfd27..1eabacb2d7a 100644
--- a/src/metabase/sync/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata.clj
@@ -8,7 +8,6 @@
4. Sync FKs and indexes
5. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"
(:require
- [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.sync.fetch-metadata :as fetch-metadata]
[metabase.sync.interface :as i]
[metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]
@@ -94,7 +93,6 @@
(if abort-cause
(sync-util/set-initial-database-sync-aborted! database abort-cause)
(sync-util/set-initial-database-sync-complete! database))
- (dependencies.hooks/after-database-sync (:id database))
results))
(mu/defn sync-db-metadata!
diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.clj
index d7171834625..f172f394788 100644
--- a/src/metabase/sync/sync_metadata/fields.clj
+++ b/src/metabase/sync/sync_metadata/fields.clj
@@ -39,6 +39,7 @@
* In general the methods in these namespaces return the number of rows updated; these numbers are summed and used
for logging purposes by higher-level sync logic."
(:require
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.sync.db :as sync.db]
@@ -94,11 +95,12 @@
table :- i/TableInstance
db-metadata :- [:set i/TableMetadataField]]
(let [db-metadata (limit-fields-to-sync table db-metadata)]
- (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
- ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
- ;; Re-fetch our metadata because there might be some things that have changed after calling
- ;; `sync-instances`
- (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table)))))
+ (dependencies.hooks/with-batched-marks
+ (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
+ ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
+ ;; Re-fetch our metadata because there might be some things that have changed after calling
+ ;; `sync-instances`
+ (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table))))))
(defn- select-best-matching-name
"Returns a key function for use with [[sort-by]] that ranks items based on how closely their `:schema` and `:name` match the given target values.
diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.clj
index b5ce3088619..1f269f9b06c 100644
--- a/src/metabase/sync/sync_metadata/tables.clj
+++ b/src/metabase/sync/sync_metadata/tables.clj
@@ -5,6 +5,7 @@
[clojure.set :as set]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.common :as lib.schema.common]
@@ -245,9 +246,10 @@
(log/info "Found new table:"
(sync-util/name-for-logging (mi/instance :model/Table table-metadata))))
(let [field-order (some-> (:engine database) driver/default-field-order)]
- (mapv (fn [table-metadata]
- (create-table! database (m/assoc-some table-metadata :field_order field-order)))
- (sort-by (juxt :schema :name) new-table-metadatas))))
+ (dependencies.hooks/with-batched-marks
+ (mapv (fn [table-metadata]
+ (create-table! database (m/assoc-some table-metadata :field_order field-order)))
+ (sort-by (juxt :schema :name) new-table-metadatas)))))
(mu/defn- retire-tables!
"Mark the active Tables with `table-ids` as inactive."
diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.clj
index 2264255649c..8208e422f76 100644
--- a/src/metabase/task/impl.clj
+++ b/src/metabase/task/impl.clj
@@ -160,6 +160,14 @@
;;; | SCHEDULING/DELETING TASKS |
;;; +----------------------------------------------------------------------------------------------------------------+
+(defn- schedule-trigger!
+ "Schedule `trigger` for `job`, scheduling `job` with it when Quartz deleted the job since it was registered."
+ [^Scheduler scheduler ^JobDetail job ^Trigger trigger]
+ (try
+ (.scheduleJob scheduler trigger)
+ (catch JobPersistenceException _
+ (.scheduleJob scheduler job trigger))))
+
(mu/defn- reschedule-task!
"Assuming that [[job]] is already registered, ensure that [[new-trigger]] is scheduled to trigger it."
[^Scheduler scheduler :- (ms/InstanceOfClass Scheduler)
@@ -173,7 +181,7 @@
replaced-trigger (or matching-trigger (first triggers))]
(log/debugf "Rescheduling job %s" (.getName job-key))
(if-not replaced-trigger
- (.scheduleJob scheduler new-trigger)
+ (schedule-trigger! scheduler job new-trigger)
(let [replaced-key (.getKey ^Trigger replaced-trigger)]
(when-not matching-trigger
(log/warnf "Replacing trigger %s with trigger %s%s"
@@ -184,7 +192,8 @@
;; Ideally we would pass *all* the new triggers at once, so we can match them up atomically.
;; The current behavior is especially confounding if replacing N triggers with M ones.
(str " (chosen randomly from " (count triggers) " existing ones)"))))
- (.rescheduleJob scheduler replaced-key new-trigger))))
+ (when-not (.rescheduleJob scheduler replaced-key new-trigger)
+ (schedule-trigger! scheduler job new-trigger)))))
(catch Throwable e
(log/errorf "Error rescheduling job: %s" (ex-message e)))))
diff --git a/src/metabase/upload/db.clj b/src/metabase/upload/db.clj
index 257081b6d53..87070d268b9 100644
--- a/src/metabase/upload/db.clj
+++ b/src/metabase/upload/db.clj
@@ -5,6 +5,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.card-schema :as queries.card-schema]
[metabase.queries.schema :as queries.schema]
+ [metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.core :as warehouse-schema]
@@ -62,20 +63,16 @@
name->display-name :- [:map-of :string :string]]
(let [user-edited (user-edited-field-names table-id (set (keys name->display-name)))
name->display-name (apply dissoc name->display-name user-edited)]
- (when (seq name->display-name)
- ;; A raw update rather than `t2/update!`, which produces an invalid query for certain versions of PostgreSQL:
- ;; SELECT * FROM "metabase_field" WHERE "id" AND ("table_id" = ?) AND ... (argument of AND must be type boolean).
- (t2/query {:update (t2/table-name :model/Field)
- :set {:display_name (into [:case]
- (mapcat identity)
- (for [[n display-name] name->display-name]
- [[:= [:lower :name] n]
- [:case
- [:= :display_name (humanization/name->human-readable-name n)] display-name
- :else :display_name]]))}
- :where [:and
- [:= :table_id table-id]
- [:in [:lower :name] (keys name->display-name)]]}))))
+ (doseq [{:keys [id name display_name]} (when (seq name->display-name)
+ (t2/select [:model/Field :id :name :display_name]
+ {:from [(warehouse-schema-overlay/field-query
+ {:user-settings? false})]
+ :where [:and
+ [:= :table_id table-id]
+ [:in [:lower :name] (keys name->display-name)]]}))
+ :let [lower-name (u/lower-case-en name)]
+ :when (= display_name (humanization/name->human-readable-name lower-name))]
+ (t2/update! :model/Field id {:display_name (name->display-name lower-name)}))))
(mu/defn mark-table-upload!
"Flag the Table with `table-id` as an authoritative, writable upload table."
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index edddb3969fd..c48f62a0c28 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -124,6 +124,7 @@
(doto :model/Field
(derive :metabase/model)
(derive :hook/timestamped?)
+ (derive :hook/dependencies)
;; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on
;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would
;; cause duplication rather than good matching if the two instances are later linked by serdes.
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 3ebc5263184..ada9dc21d90 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -106,7 +106,8 @@
(doto :model/FieldValues
(derive :metabase/model)
- (derive :hook/timestamped?))
+ (derive :hook/timestamped?)
+ (derive :hook/dependencies))
(t2/deftransforms :model/FieldValues
{:human_readable_values mi/transform-json-no-keywordization
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 48c984bbd92..322d6b62397 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -94,6 +94,7 @@
(derive ::mi/read-policy.full-perms-for-perms-set)
(derive ::mi/write-policy.full-perms-for-perms-set)
(derive :hook/timestamped?)
+ (derive :hook/dependencies)
;; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on
;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would
;; cause duplication rather than good matching if the two instances are later linked by serdes.
@@ -255,13 +256,12 @@
(cond
;; active: true -> false (table being deactivated)
(and (true? current-active) (false? new-active))
- (assoc changes :deactivated_at (mi/now))
+ (merge table changes {:deactivated_at (mi/now)})
;; active: false -> true (table being reactivated)
(and (false? current-active) (true? new-active))
- (assoc changes
- :deactivated_at nil
- :archived_at nil)
+ (merge table changes {:deactivated_at nil
+ :archived_at nil})
:else (merge table changes)))))
diff --git a/test/metabase/app_db/quartz_test.clj b/test/metabase/app_db/quartz_test.clj
index 7f0c0a618c4..17a89cdb748 100644
--- a/test/metabase/app_db/quartz_test.clj
+++ b/test/metabase/app_db/quartz_test.clj
@@ -176,6 +176,7 @@
"metabase_enterprise.data_complexity_score.task.complexity_score_trimmer.DataComplexityScoreTrimmer"
"metabase_enterprise.dependencies.task.backfill.BackfillDependencies"
"metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"
+ "metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata"
"metabase_enterprise.entity_retrieval.task.sync.OsiAiContextSync"
"metabase_enterprise.mcp.task.mcp_usage_trimmer.McpUsageTrimmer"
"metabase_enterprise.metabot.task.ai_usage_trimmer.AiUsageTrimmer"
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index e13f89b63d4..8e287bc75bf 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -139,6 +139,7 @@
:model/AiUsageLog
:model/AnalysisFinding
:model/AnalysisFindingError
+ :model/AnalysisFindingStatus
:model/ApiKeyUsageLog
:model/CacheConfig
:model/CardFavorite
@@ -167,6 +168,7 @@
:model/RemoteSyncObject
:model/RemoteSyncTask
:model/ReplacementRun
+ :model/ResultMetadataStatus
:model/SearchIndexMetadata
:model/SecurityAdvisory
:model/SemanticSearchTokenTracking
diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.clj
index 4625a19d909..79f749a0772 100644
--- a/test/metabase/queries/models/card/metadata_test.clj
+++ b/test/metabase/queries/models/card/metadata_test.clj
@@ -5,6 +5,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.queries.models.card.metadata :as card.metadata]
[metabase.queries.schema :as queries.schema]
+ [metabase.query-processor :as qp]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -48,8 +49,8 @@
:base_type :type/BigInteger}]}
(card.metadata/populate-result-metadata (lib/normalize ::queries.schema/card card)))))))
-(deftest ^:parallel infer-metadata-no-remaps
- (testing "infer-metadata should not include remapped columns (#67128)"
+(deftest infer-metadata-remaps-test
+ (testing "infer-metadata returns the remapped columns of a run, which a query on the card leaves out (#67128)"
(mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :user_id)
:name "User ID"
:human_readable_field_id (mt/id :people :name)
@@ -58,10 +59,26 @@
:name "Product ID"
:human_readable_field_id (mt/id :products :title)
:type :external}]
- (let [mp (mt/metadata-provider)
- query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
- (is (= 9 ;; there are two remaps, giving 11 if included
- (count (card.metadata/infer-metadata query))))))))
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ metadata (card.metadata/infer-metadata query)
+ run (get-in (qp/process-query query) [:data :results_metadata :columns])
+ remap (juxt :name :remapped_from :remapped_to)]
+ (is (= (mapv remap run) (mapv remap metadata)))
+ (is (= 2 (count (filter :remapped_from metadata))))
+ (mt/with-temp [:model/Card {card-id :id} {:type :model :dataset_query query :result_metadata metadata}]
+ (let [mp (mt/metadata-provider)]
+ (is (= (mapv :name (lib/returned-columns query))
+ (mapv :name (lib/returned-columns (lib/query mp (lib.metadata/card mp card-id))))))))))))
+
+(deftest infer-metadata-internal-remaps-test
+ (testing "infer-metadata returns the column of an internal remap that a run adds"
+ (mt/with-column-remappings [venues.category_id (values-of categories.name)]
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ remap (juxt :name :display_name :remapped_from :remapped_to :base_type)]
+ (is (= (mapv remap (get-in (qp/process-query query) [:data :results_metadata :columns]))
+ (mapv remap (card.metadata/infer-metadata query))))))))
(deftest ^:parallel untyped-metadata?-test
(testing "metadata is untyped when it has columns and none of them has a known base type (GHY-4213)"
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index d1caead3b34..0141c54504f 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1836,7 +1836,7 @@
(is (= :question
(t2/select-one-fn :type :model/Card :id (:id card)))))))
-(deftest create-card-no-remaps
+(deftest create-card-remaps-test
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :user_id)
:name "User ID"
@@ -1856,8 +1856,8 @@
:dataset_query query}
@api/*current-user*)]
(try
- (is (= 9
- (count (:result_metadata card))))
+ (testing "a created card stores the remapped columns of a run"
+ (is (= 2 (count (filter :remapped_from (:result_metadata card))))))
(finally
(t2/delete! :model/Card (:id card))))))))
@@ -1911,9 +1911,9 @@
:name "Orders Model for remap test"
:dataset_query query}
@api/*current-user*)]
- (testing "model result_metadata should not include remap columns"
- (is (not-any? #(re-find #"(?i)remap" (:name %))
- (:result_metadata card))))
+ (testing "model result_metadata includes the remap column of a run, marked as remapped"
+ (is (=? [{:remapped_from "QUANTITY"}]
+ (filter #(re-find #"(?i)remap" (:name %)) (:result_metadata card)))))
(let [mp (mt/metadata-provider)
model-query (-> (lib/query mp (lib.metadata/card mp (:id card)))
(lib/aggregate (lib/count))
diff --git a/test/metabase/task_test.clj b/test/metabase/task_test.clj
index 04a6d721cb6..46ef792c60e 100644
--- a/test/metabase/task_test.clj
+++ b/test/metabase/task_test.clj
@@ -103,6 +103,20 @@
:misfire-instruction CronTrigger/MISFIRE_INSTRUCTION_IGNORE_MISFIRE_POLICY}}
(triggers))))))
+(deftest reschedule-after-job-was-deleted-test
+ (testing "rescheduling schedules the new trigger when the job and its trigger are deleted while it reschedules, as
+ Quartz does when the last run of a one-shot trigger completes"
+ (with-temp-scheduler-and-cleanup!
+ (task/schedule-task! (job) (trigger-1))
+ (let [get-triggers (mt/original-fn #'qs/get-triggers-of-job)]
+ (mt/with-dynamic-fn-redefs [qs/get-triggers-of-job (fn [scheduler job-key]
+ (u/prog1 (get-triggers scheduler job-key)
+ (qs/delete-job scheduler job-key)))]
+ (task/schedule-task! (job) (trigger-2))))
+ (is (= #{{:cron-expression "0 0 6 * * ? *"
+ :misfire-instruction CronTrigger/MISFIRE_INSTRUCTION_IGNORE_MISFIRE_POLICY}}
+ (triggers))))))
+
(deftest scheduler-info-test
(testing "Make sure scheduler-info doesn't explode and returns info in the general shape we expect"
(mt/with-temp-scheduler!