The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Re-derive the result metadata of MBQL cards from the dependency graph (#84475)
Alexander Polyankin3e8f2152034f
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 4a26230229e..69ffb6cfb88 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3906,6 +3906,7 @@lib.metadatalib.metriclib.schema+ lib.typesmeasuresmodelsnative-query-snippets@@ -3937,6 +3938,7 @@:model/Document:model/Field:model/FieldUserSettings+ :model/FieldValues:model/Measure:model/NativeQuerySnippet:model/QueryAction@@ -4427,7 +4429,6 @@app-dbappearancecollections- dependenciesliblib.schemaloggerdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 1d4b2a407f4..4523030f1e2 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -25,7 +25,7 @@:discouraged-java-method 4:duplicate-require 1:main-without-gen-class 1- :metabase/discourage-dynamic-vars 325+ :metabase/discourage-dynamic-vars 326:metabase/discourage-millis-duration 1:metabase/modules 23:metabase/namespace-name 3diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.cljnew file mode 100644index 00000000000..c7a44f37411--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj@@ -0,0 +1,144 @@+(ns metabase-enterprise.dependencies.analysis-finding+ "Analyzes the entities whose AnalysisFindingStatus a write marked stale, recording what each one's query no longer finds."+ (:require+ [metabase-enterprise.dependencies.analysis :as deps.analysis]+ [metabase-enterprise.dependencies.db :as dependencies.db]+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.schema :as dependencies.schema]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defmulti ^:private entity-db-id+ "The database ID of `entity`."+ {:arglists '([entity])}+ t2/model)++(defmethod entity-db-id :default+ [entity]+ (:database_id entity))++(defmethod entity-db-id :model/Transform+ [entity]+ (or (some-> entity :source :query lib/database-id)+ ;; non-query transforms (e.g. Python) have no query to read the database from+ (:source_database_id entity)))++(defmethod entity-db-id :model/Segment+ [entity]+ (some-> entity :definition lib/database-id))++(defmulti ^:private pre-analysis-errors+ "The set of errors of `entity` found without running the analysis, such as an orphan Transform's, which when not+ empty are recorded instead of running it."+ {:arglists '([entity])}+ t2/model)++(defmethod pre-analysis-errors :default [_] nil)++(defmethod pre-analysis-errors :model/Transform+ [{db-id :source_database_id id :id}]+ (when (nil? db-id)+ #{(-> (lib/validation-exception-error "Source database for this transform has been deleted.")+ (assoc :source-entity-type :transform :source-entity-id id))}))++(mu/defn upsert-analysis!+ "Analyze the Toucan entity `entity` and insert or replace its AnalysisFinding, reusing the caller's metadata provider+ cache when it has one."+ [entity :- [:fn {:error/message "a Toucan instance"} t2/instance?]]+ (lib-be/with-metadata-provider-cache+ (let [model (t2/model entity)+ entity-type (deps.dependency-types/model->dependency-type model)+ entity-id (:id entity)]+ (if-let [errors (not-empty (pre-analysis-errors entity))]+ (models.analysis-finding/upsert-analysis! entity-type entity-id false errors)+ (if-let [db-id (entity-db-id entity)]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ results (try (deps.analysis/check-entity mp entity-type entity-id)+ (catch Exception e+ (log/errorf "Error analyzing entity: %s" (ex-message e))+ #{(lib/validation-exception-error (.getMessage e))}))+ success (empty? results)]+ (models.analysis-finding/upsert-analysis! entity-type entity-id success results))+ (models.analysis-finding/upsert-analysis!+ entity-type entity-id false+ #{(lib/validation-exception-error "Could not resolve a database for this entity.")}))))))++(mu/defn- analyze-entity!+ "Analyze `entity`, recording a failure as a failed finding before rethrowing it."+ [entity :- [:fn {:error/message "a Toucan instance"} t2/instance?]]+ (try (upsert-analysis! entity)+ (catch Exception e+ (log/errorf "Analyzing entity %s %s failed: %s"+ (t2/model entity) (:id entity) (ex-message e))+ (try+ (models.analysis-finding/upsert-analysis!+ (deps.dependency-types/model->dependency-type (t2/model entity)) (:id entity) false+ #{(lib/validation-exception-error (ex-message e))})+ (catch Exception record-e+ (log/errorf "Recording the failure of entity %s %s failed: %s"+ (t2/model entity) (:id entity) (ex-message record-e))))+ (throw e))))++(def analyzable-entities+ "Entities for which we can compute analysis findings."+ #{:card :transform :segment})++(mu/defn- remove-deleted-statuses! :- nat-int?+ "Delete the AnalysisFindingStatuses of up to `batch-size` deleted entities of each analyzable type, returning how+ many it deleted."+ [batch-size :- pos-int?]+ (reduce + (for [entity-type (sort analyzable-entities)+ :let [ids (dependencies.db/analysis-finding-deleted-entity-ids entity-type batch-size)]]+ (if (seq ids)+ (dependencies.db/delete-analysis-finding-statuses! entity-type ids)+ 0))))++(mu/defn- take-pending-statuses! :- [:map-of ::deps.dependency-types/dependency-types+ [:sequential ::dependencies.schema/pending-status]]+ "Up to `batch-size` analyzable entities whose AnalysisFindingStatus counts a write, is missing or is outdated, as+ `{entity-type statuses}`, each with the `stale` count the job clears afterwards."+ [batch-size :- pos-int?]+ (reduce (fn [taken entity-type]+ (let [remaining (- batch-size (reduce + (map count (vals taken))))+ statuses (when (pos? remaining)+ (dependencies.db/take-pending-analysis-finding-statuses!+ entity-type remaining models.analysis-finding/*current-analysis-finding-version*+ models.dependency/current-dependency-analysis-version))]+ (cond-> taken+ (seq statuses) (assoc entity-type statuses))))+ {}+ (sort analyzable-entities)))++(mu/defn analyze-batch! :- nat-int?+ "Analyze up to `batch-size` analyzable entities whose AnalysisFindingStatus counts a write, is missing or is outdated,+ clearing each one's `stale` count unless a write raised it meanwhile, or counting its failure, returning how many it+ took."+ [batch-size :- pos-int?]+ ;; The cache spans the select too: reading an entity attaches a `MetadataProvider` to its query+ ;; (see `metabase.lib-be.models.transforms/transform-query`), so selecting outside the cache gives every entity in+ ;; the batch a private one.+ (let [removed (remove-deleted-statuses! batch-size)]+ (lib-be/with-metadata-provider-cache+ (let [taken (take-pending-statuses! batch-size)]+ (doseq [[entity-type statuses] taken+ :let [stale (into {} (map (juxt :entity_id :stale)) statuses)]+ entity (dependencies.db/entities entity-type (set (keys stale)))]+ (try+ (analyze-entity! entity)+ (dependencies.db/clear-analysis-finding-status!+ entity-type (:id entity) (stale (:id entity)) models.analysis-finding/*current-analysis-finding-version*)+ (catch Throwable e+ (when (dependencies.db/record-analysis-finding-failure! entity-type (:id entity)+ (deps.settings/dependency-max-failures))+ (log/warnf "Giving up on analyzing %s %s until it changes" (name entity-type) (:id entity)))+ (when-not (instance? Exception e)+ (throw e)))))+ (+ removed (reduce + (map count (vals taken))))))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 5800c7240e7..c81fb4f9816 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -6,6 +6,7 @@[metabase-enterprise.dependencies.models.analysis-finding-error :as analysis-finding-error][metabase-enterprise.dependencies.models.dependency :as dependency][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.schema :as deps.schema][metabase-enterprise.sandbox.schema][metabase.actions.schema :as actions.schema][metabase.api.common :as api]@@ -53,7 +54,13 @@:measure [:name :description :created_at :creator :creator_id :table :table_id]:action [:name :description :type :model_id :parameters :created_at :creator :creator_id :collection :collection_id]})-(defn- format-subentity [entity]+(mr/def ::data-value+ [:schema {::mr/deliberately-open true, :description "any column or hydrated value of a dependency-tracked entity"}+ :any])++(mu/defn- format-subentity :- ::data-value+ "`entity` slimmed to the keys the response shows when it is a hydrated Collection, Dashboard, or Document."+ [entity :- ::data-value](case (t2/model entity):model/Collection (select-keys entity [:id :name :authority_level :is_personal :namespace :location]):model/Dashboard (select-keys entity [:id :name])@@ -65,16 +72,32 @@[:enum :table :snippet :transform :dashboard :document :sandbox :segment :question :model :metric :measure :action]::deps.dependency-types/entity-id])+(mr/def ::normalized-finding-error+ "The slim `{:type :detail}` shape [[normalize-finding-error]] produces from an AnalysisFindingError."+ [:map {:closed true}+ [:type ::lib.schema.validate/validate-error-type]+ [:detail {:optional true} [:maybe :string]]])+(mr/def ::dependent-error"One entry of `::base-entity`'s `:dependents_errors`: either the slim `{:type :detail}` shape[[normalize-finding-error]] produces for an entity's own errors (`node-errors`), or the raw AnalysisFindingError row`node-downstream-errors` keeps so callers can tell which downstream entity each error belongs to."[:or- [:map {:closed true}- [:type ::lib.schema.validate/validate-error-type]- [:detail {:optional true} [:maybe :string]]]+ ::normalized-finding-error::analysis-finding-error/analysis-finding-error])+(mr/def ::nodes-by-type+ "Entity ids grouped by entity type."+ [:map-of ::deps.dependency-types/dependency-types [:sequential ::deps.dependency-types/entity-id]])++(mr/def ::node->usages+ "The dependent counts of each node."+ [:map-of ::deps.schema/node ::usages])++(mr/def ::node->errors+ "The errors of each node."+ [:map-of ::deps.schema/node [:set ::dependent-error]])+(mr/def ::base-entity[:map[:id pos-int?]@@ -83,10 +106,10 @@[:dependents_count [:maybe [:ref ::usages]]][:dependents_errors {:optional true} [:set [:ref ::dependent-error]]]])-(defn- fields-for [entity-key]- ;; these specs should really use something like- #_[:data [:select-keys [:ref :blah/table] (entity-keys :table)]]- ;; but :select-keys seems to mess up open-api spec generation+(mu/defn- fields-for :- [:cat [:= :map] [:* [:tuple :keyword [:map {:closed true} [:optional [:= true]]] [:= :any]]]]+ "An open map schema with every key of `entity-key`'s `entity-keys` optional; not a `:select-keys` schema, which+ breaks the OpenAPI spec generation."+ [entity-key :- (into [:enum] (keys entity-keys))](into [:map](map (fn [key] [key {:optional true} :any]))(entity-keys entity-key)))@@ -174,6 +197,26 @@[:measure [:ref ::measure-entity]][:action [:ref ::action-entity]]])+(mr/def ::entity-data+ "The `:data` of an `::entity` as the helpers that filter and sort entities read it: the keys of `entity-keys`, each+ optional."+ (into [:map {:closed true}]+ (map (fn [k]+ [k {:optional true} (case k+ (:name :display_name) [:maybe :string]+ :view_count [:maybe :int]+ ::data-value)]))+ (distinct (mapcat val entity-keys))))++(mr/def ::entity-node+ "An `::entity`, or a `::broken-dependent-entity`, as the helpers that filter and sort entities read it."+ [:map {:closed true}+ [:id ::deps.dependency-types/entity-id]+ [:type ::deps.dependency-types/dependency-types]+ [:data ::entity-data]+ [:dependents_count {:optional true} [:maybe [:ref ::usages]]]+ [:dependents_errors {:optional true} [:set [:ref ::dependent-error]]]])+(def ^:private Entity"Any dependency-tracked entity `entity-value` accepts, hydrated the way `hydrate-entities` hydrates it."[:or@@ -191,12 +234,8 @@(mu/defn- entity-value :- ::entity[entity-type :- ::deps.dependency-types/dependency-types{:keys [id] :as entity} :- Entity- usages :- [:maybe [:map-of- [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]- ::usages]]- errors :- [:maybe [:map-of- [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]- [:set ::dependent-error]]]]+ usages :- [:maybe ::node->usages]+ errors :- [:maybe ::node->errors]](cond-> {:id id:type entity-type:data (-> (select-keys entity (entity-keys entity-type))@@ -233,34 +272,55 @@:measure [:id :name :description :created_at :creator_id :table_id]:action [:id :name :description :type :model_id :parameters :created_at :creator_id :collection_id]})-(defn- current-user-visibility+(mr/def ::include-archived-items+ [:enum :exclude :all :only])++(mr/def ::visible-opts+ "The `:visible` filter-spec opts consumed by `metabase-enterprise.dependencies.db`."+ [:map {:closed true}+ [:user-id ::lib.schema.id/user]+ [:is-superuser? {:optional true} [:maybe :boolean]]+ [:is-data-analyst? {:optional true} [:maybe :boolean]]+ [:include-archived-items {:optional true} ::include-archived-items]])++(mu/defn- current-user-visibility :- ::visible-opts"The current user, as the `:visible` filter-spec opts consumed by `metabase-enterprise.dependencies.db`.Analyst-wide visibility comes with the role rather than a permissions graph, so it pauses while the`advanced-permissions` feature is unavailable (see [[api/entitled-data-analyst?]])."- [{:keys [include-archived-items]}]+ [{:keys [include-archived-items]} :- [:maybe [:map {:closed true}+ [:include-archived-items {:optional true}+ [:maybe ::include-archived-items]]]]](cond-> {:user-id api/*current-user-id*:is-superuser? api/*is-superuser?*:is-data-analyst? (api/entitled-data-analyst?)}include-archived-items (assoc :include-archived-items include-archived-items)))-(defn- readable-graph-dependencies+(mu/defn- readable-graph-dependencies :- ::graph/graph+ "The upstream dependency graph, restricted to the entities the current user can see."([](readable-graph-dependencies nil))- ([opts]+ ([opts :- [:maybe [:map {:closed true}+ [:include-archived-items {:optional true} [:maybe ::include-archived-items]]]]](dependency/filtered-graph-dependencies {:visible (current-user-visibility opts)})))-(defn- readable-graph-dependents+(mu/defn- readable-graph-dependents :- ::graph/graph+ "The downstream dependents graph, restricted to the entities the current user can see, and to broken ones when+ `broken`."([](readable-graph-dependents nil))- ([{:keys [include-archived-items broken] :or {include-archived-items :exclude}}]+ ([{:keys [include-archived-items broken] :or {include-archived-items :exclude}}+ :- [:maybe [:map {:closed true}+ [:include-archived-items {:optional true} [:maybe ::include-archived-items]]+ [:broken {:optional true} [:maybe :boolean]]]]](dependency/filtered-graph-dependents(cond-> {:visible (current-user-visibility {:include-archived-items include-archived-items})}broken (assoc :broken? true)))))-(defn- node-usages- "Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`. "- [graph nodes]+(mu/defn- node-usages :- ::node->usages+ "Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`."+ [graph :- ::graph/graph+ nodes :- [:sequential ::deps.schema/node]](let [children-map (graph/children-of graph nodes)all-cards (->> (vals children-map)(apply concat)@@ -279,18 +339,19 @@{dependency-type 1})))(apply merge-with +))))))-(defn- normalize-finding-error- [{:keys [error_type error_detail]}]+(mu/defn- normalize-finding-error :- ::normalized-finding-error+ "The error type and detail of an AnalysisFindingError."+ [{:keys [error_type error_detail]} :- ::analysis-finding-error/analysis-finding-error](cond-> {:type error_type}error_detail (assoc :detail error_detail)))-(defn- node-downstream-errors+(mu/defn- node-downstream-errors :- [:maybe ::node->errors]"Fetches errors caused by the given source entities (what downstream entities they're breaking).Filters out errors where the analyzed entity is not visible to the current user.Unlike `node-errors` which fetches errors on an entity, this fetches errors thatthe entity is causing in other entities that depend on it. Keeps the raw AnalysisFindingError shape (unlike`node-errors`'s normalized one) so callers can tell which downstream entity each error belongs to."- [nodes-by-type]+ [nodes-by-type :- ::nodes-by-type](letfn [(errors-by-source-type-and-id [[source-type ids]](when (seq ids)(let [finding-errors (dependencies.db/finding-errors-from-sources@@ -301,11 +362,11 @@(into {} (mapcat errors-by-source-type-and-id))not-empty)))-(defn- node-errors+(mu/defn- node-errors :- [:maybe ::node->errors]"Fetches and normalizes AnalysisFindingErrors for the given entities.Filters out errors where the source entity is not visible to the current user.Returns {[entity-type entity-id] #{error-maps...}}, or nil if none."- [nodes-by-type]+ [nodes-by-type :- ::nodes-by-type](letfn [(errors-by-entity-type-and-id [[type ids]](when (seq ids)(let [finding-errors (dependencies.db/finding-errors-for-entities-with-visible-sources@@ -316,7 +377,10 @@(into {} (mapcat errors-by-entity-type-and-id))not-empty)))-(defn- hydrate-entities [entity-type entities]+(mu/defn- hydrate-entities :- [:sequential Entity]+ "`entities` of the entity type `entity-type`, hydrated with what the dependency graph responses show."+ [entity-type :- ::deps.dependency-types/dependency-types+ entities :- [:sequential Entity]](case entity-type:card (-> entities(t2/hydrate :creator :dashboard :document [:collection :is_personal])@@ -344,21 +408,27 @@(collection.root/root-collection-with-ui-details(dependencies.db/action-root-namespace (nil? (:model_id %)))))))))-(defn- fetch-and-hydrate-nodes+(mu/defn- fetch-and-hydrate-nodes :- [:map-of ::deps.schema/node Entity]"Fetches and hydrates entities for the given nodes.Returns a map from [entity-type entity-id] -> hydrated entity."- [nodes-by-type]+ [nodes-by-type :- ::nodes-by-type](into {}(mapcat (fn [[entity-type entity-ids]](when (seq entity-ids)(let [fields (entity-select-fields entity-type)]- (->> (dependencies.db/instances-with-columns entity-type fields entity-ids)+ (->> (dependencies.db/entities-with-columns entity-type fields entity-ids)(hydrate-entities entity-type)(map (fn [entity][[entity-type (:id entity)] entity])))))))nodes-by-type))-(defn- expanded-nodes [downstream-graph nodes {:keys [include-errors?]}]+(mu/defn- expanded-nodes :- [:sequential ::entity]+ "The `::entity` of each of `nodes` the current user can see, with its dependent counts from `downstream-graph`, and+ its errors when `include-errors?`."+ [downstream-graph :- ::graph/graph+ nodes :- [:sequential ::deps.schema/node]+ {:keys [include-errors?]} :- [:map {:closed true}+ [:include-errors? :boolean]]](let [usages (node-usages downstream-graph nodes)nodes-by-type (-> (group-by first nodes)(update-vals #(map second %)))@@ -394,8 +464,8 @@upstream-graph (readable-graph-dependencies {:include-archived-items :all})downstream-graph (graph/cached-graph (readable-graph-dependents))edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))- nodes (into (set starting-nodes)- (graph/transitive upstream-graph starting-nodes))+ nodes (vec (into (set starting-nodes)+ (graph/transitive upstream-graph starting-nodes)))edges (graph/edges-between edge-graph nodes)]{:nodes (expanded-nodes downstream-graph nodes {:include-errors? false}):edges edges}))@@ -408,18 +478,18 @@"Valid sort columns for the /graph/dependents endpoint."#{:name :location :view-count})-(defn- entity-name+(mu/defn- entity-name :- [:maybe :string]"Returns the name string for an entity based on its type. Might return nil."- [entity]+ [entity :- ::entity-node](let [data (:data entity)](case (:type entity):table (:display_name data):sandbox nil(:name data))))-(defn- entity-location+(mu/defn- entity-location :- [:maybe :string]"Returns the location string for an entity based on its type. Might return nil."- [entity]+ [entity :- ::entity-node](let [data (:data entity)](case (:type entity):card (or (-> data :dashboard :name)@@ -431,27 +501,31 @@(:segment :measure) (-> data :table :display_name)nil)))-(defn- string-matches-query?+(mu/defn- string-matches-query? :- [:maybe :boolean]"Returns true if `s` is a string and its lower-case version contains the string `query`.`query` is expected to be lower-case."- [s query]+ [s :- [:maybe :string]+ query :- :string](some-> s u/lower-case-en (str/includes? query)))-(defn- entity-matches-query?+(mu/defn- entity-matches-query? :- [:maybe :boolean]"Returns true if `entity`'s name or location contains the string `query` (case-insensitive)."- [entity query]+ [entity :- ::entity-node+ query :- :string](let [q (u/lower-case-en query)](or (string-matches-query? (entity-name entity) q)(string-matches-query? (entity-location entity) q))))-(defn- in-personal-collection?+(mu/defn- in-personal-collection? :- [:maybe :boolean]"Returns true if `entity` is in a personal collection."- [entity]+ [entity :- ::entity-node](get-in entity [:data :collection :is_personal]))-(defn- sort-dependents+(mu/defn- sort-dependents :- [:sequential ::entity-node]"Sort `entities` by `sort-column` in `sort-direction`."- [entities sort-column sort-direction]+ [entities :- [:sequential ::entity-node]+ sort-column :- (into [:enum] dependents-sort-columns)+ sort-direction :- (into [:enum] sort-directions)](let [key-fn (case sort-column:name #(some-> (entity-name %) u/lower-case-en):location #(some-> (entity-location %) u/lower-case-en)@@ -503,8 +577,8 @@sort-direction :asc}} :- dependents-args](api/read-check (deps.dependency-types/dependency-type->model type) id)(let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))- nodes (-> (graph/children-of downstream-graph [[type id]])- (get [type id]))+ nodes (vec (-> (graph/children-of downstream-graph [[type id]])+ (get [type id])))dep-types-set (cond(nil? dependent-types) deps.dependency-types/dependency-types(sequential? dependent-types) (set dependent-types)@@ -658,7 +732,7 @@usages (node-usages downstream-graph all-ids)fetch-entity (fn [entity-type entity-id](let [fields (entity-select-fields entity-type)]- (dependencies.db/instance-with-columns entity-type fields entity-id)))+ (dependencies.db/entity-with-columns entity-type fields entity-id)))data (into [](keep (fn [[entity-type entity-id]](when-let [entity (fetch-entity entity-type entity-id)]@@ -742,10 +816,10 @@(api.macros/defendpoint :get "/backfill-status" :- [:map[:complete :boolean]]- "Returns whether the dependency backfill has pending work.- `complete` is true when there are no stale or outdated entities awaiting processing."+ "Whether no entity awaits the computation of its dependencies, as `complete`, for superusers."[_route-params _query-params]- {:complete (not (deps.dependency-status/has-stale-or-outdated?))})+ (api/check-superuser)+ {:complete (not (deps.dependency-status/has-pending-entities?))})(def ^{:arglists '([request respond raise])} routes"`/api/ee/dependencies` routes."diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex b0288749f1a..ce1e9ab49cd 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -67,7 +67,7 @@(let [driver (:engine (lib.metadata/database query))deps (deps.native/native-query-deps driver query)];; The deps are in #{{:table 7} ...} form and need conversion to ::deps.schema/upstream-deps form.- (u/group-by ffirst (comp second first) conj #{} deps)))+ (u/group-by ffirst (comp second first) conj #{} (remove :unresolved-table deps))))(mu/defn- upstream-deps:query :- ::deps.schema/upstream-deps[query :- ::lib.schema/query]@@ -86,6 +86,7 @@;; Modified implementation of documents.models.document/document-deps(defn- document-deps+ "The `[entity-type entity-id]` pairs that the smart links and card embeds of a prose-mirror `document` point at."[{:keys [content_type] :as document}](when (= content_type prose-mirror/prose-mirror-content-type)(prose-mirror/collect-ast document (fn [{:keys [type attrs]}]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.cljindex 45d04b5c9b8..75f904983e5 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj@@ -7,17 +7,20 @@[metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.metadata-provider :as deps.provider][metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase-enterprise.dependencies.schema :as deps.schema][metabase.graph.core :as graph][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.validate :as lib.schema.validate][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]))(mr/def ::entity-type[:enum :card :transform :snippet :table])@@ -29,8 +32,14 @@[:snippet {:optional true} [:sequential ::lib.schema.metadata/native-query-snippet]][:table {:optional true} [:sequential ::lib.schema.metadata/table]]])-(defn- transitive-dependents- [& {:keys [graph updated-entities include-native?]}]+(mu/defn- transitive-dependents :- ::deps.schema/type->ids+ "The transitive dependents of `updated-entities` in `graph` (the app DB by default), leaving out native entities and+ their children unless `include-native?`."+ [& {:keys [graph updated-entities include-native?]}+ :- [:map {:closed true}+ [:graph {:optional true} [:maybe ::graph/graph]]+ [:updated-entities ::updates-map]+ [:include-native? {:optional true} [:maybe :boolean]]]](if include-native?(deps.graph/transitive-dependents graph updated-entities)(deps.graph/transitive-mbql-dependents graph updated-entities)))@@ -66,7 +75,7 @@:updated-entities updated-entities:dependent-ids dependents}))))-(mu/defn- check-query-soundness :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error]]]]+(mu/defn- check-query-soundness :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error-with-source]]]]"Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependentsto check that everything is still sound.@@ -92,7 +101,10 @@(vswap! errors assoc-in [entity-type id] bad-refs)))@errors))-(defn- group-by-db [deps]+(mu/defn- group-by-db :- [:map-of ::lib.schema.id/database [:map-of ::entity-type [:set ms/PositiveInt]]]+ "The card, table, transform, and snippet ids of `deps` grouped by the id of the Database each belongs to, with the+ snippets copied to every Database."+ [deps :- ::deps.schema/type->ids](let [by-db (volatile! {})](when (seq (:card deps))(doseq [[db-id card-ids] (->> (dependencies.db/card-database-ids (:card deps))@@ -113,7 +125,7 @@(vswap! by-db update-vals #(assoc % :snippet (:snippet deps))))@by-db))-(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error]]]]+(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:set [:ref ::lib.schema.validate/error-with-source]]]]"Given a regular `MetadataProvider`, and a map of entity types (`:card`, `:transform`, `:snippet`) to lists ofupdated entities, this returns a map of `{entity-type {entity-id [bad-ref ...]}}`.@@ -133,9 +145,9 @@(let [valid-edits (if include-native?edits(into {}- (map (fn [[entity-type instances]]+ (map (fn [[entity-type entities]][entity-type (remove #(deps.graph/is-native-entity? entity-type %)- instances)]))+ entities)]))edits))](cond(not (some seq (vals valid-edits)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.cljindex ee424bdee36..b413983f665 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj@@ -1,6 +1,6 @@(ns metabase-enterprise.dependencies.db- "Application database queries for the dependencies module. Every function here is a direct Toucan 2 call with no- additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods, and transactions."+ "Application database queries for the dependencies module, so the rest of the module only touches `toucan2.core` for+ model definitions, hydration methods, and transactions."(:require[clojure.set :as set][malli.util :as mut]@@ -10,48 +10,34 @@[metabase.collections.models.collection :as collection][metabase.collections.models.collection.root :as collection.root][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi][metabase.permissions.core :as perms]+ [metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))-(def ^:private VisibleOpts- "Opts consumed by [[visible-entities-expr]]: the current user (`:user-id`, `:is-superuser?`,- `:is-data-analyst?`), plus whether to include archived items (`:include-archived-items`,- default `:exclude`)."- [:map {:closed true}- [:user-id ::lib.schema.id/user]- [:is-superuser? {:optional true} [:maybe :boolean]]- [:is-data-analyst? {:optional true} [:maybe :boolean]]- [:include-archived-items {:optional true} [:enum :exclude :all :only]]])--(def ^:private EntityType- "A dependency-type keyword, or the equivalent string. Every column these entity types are filtered against uses- Toucan's keyword transform, which coerces either representation, so callers (and tests) sometimes pass a plain- string instead of the keyword."- [:or ::deps.dependency-types/dependency-types :string])--(def ^:private RestrictionSpec- "The edge-restriction filter spec consumed by [[edge-restriction-expr]]: nil for no restriction, or a map- combining `:visible` ([[VisibleOpts]]), `:broken?`, `:types` (a set of dependency types), and/or- `:entity-type` + `:ids` (restricting to specific entities of one type)."- [:maybe- [:map {:closed true}- [:visible {:optional true} VisibleOpts]- [:broken? {:optional true} :boolean]- [:types {:optional true} [:set EntityType]]- [:entity-type {:optional true} EntityType]- [:ids {:optional true} [:sequential ::deps.dependency-types/entity-id]]]])+(def ^:private Column+ "A Honey SQL column keyword, or a string literal standing in for one."+ [:or :keyword :string])++(def ^:private Join+ "A LEFT JOIN [[build-left-joins]] adds."+ [:enum :collection :dashboard :database :document :table])++(def ^:private HoneySQLForm+ "A Honey SQL expression or query map."+ [:or ::h2x/honeysql-expr ::h2x/honeysql-clause-opts]);;; ------------------------------------------------ Graph edge restrictions ------------------------------------------------;;; The dependency graph is traversed generically over every entity type the module knows about; these private;;; helpers build the permission and type-allowlist fragments the graph and item-list queries below restrict edges;;; and items by.-(defn- visible-entities-expr+(mu/defn- visible-entities-expr :- ::h2x/honeysql-expr"Matches entities at `entity-type-field`/`entity-id-field` that are readable by the user described by `user-id`,`is-superuser?`, and `is-data-analyst?`, honoring `include-archived-items` (`:exclude`, `:all`, or `:only`,default `:exclude`; applies to both archived collections and archived entities).@@ -65,12 +51,14 @@- Table: filters by table visibility permissions. Tables are NOT filtered by active/visibility_type regardlessof `include-archived-items`, so dependencies broken by dropped or hidden tables stay visible.- Transform: analysts can view any transform they have source view permission to."- [entity-type-field entity-id-field {:keys [user-id is-superuser? is-data-analyst? include-archived-items]- :or {include-archived-items :exclude}}]+ [entity-type-field :- Column+ entity-id-field :- Column+ {:keys [user-id is-superuser? is-data-analyst? include-archived-items]+ :or {include-archived-items :exclude}} :- ::dependencies.schema/visible-opts](into [:or](keep (fn [[entity-type model]](let [table-name (t2/table-name model)- id-column (keyword (name table-name) "id")]+ id-column (u/qualified-key table-name :id)](case model:model/Sandbox(when is-superuser?@@ -100,10 +88,10 @@{:perms/create-queries :query-builder})]}]])(:model/Card :model/Dashboard :model/Document :model/NativeQuerySnippet :model/Action)- (let [archived-column (keyword (name table-name) "archived")+ (let [archived-column (u/qualified-key table-name :archived)visible-in-root (fn [root-namespace](collection/visible-collection-filter-clause- (keyword (name table-name) "collection_id")+ (u/qualified-key table-name :collection_id){:include-archived-items include-archived-items:root-namespace root-namespace}{:current-user-id user-id@@ -147,8 +135,8 @@{:perms/view-data :unrestricted :perms/create-queries :query-builder})]}]](:model/Segment :model/Measure)- (let [archived-column (keyword (name table-name) "archived")- table-id-column (keyword (name table-name) "table_id")]+ (let [archived-column (u/qualified-key table-name :archived)+ table-id-column (u/qualified-key table-name :table_id)][:and[:= entity-type-field (name entity-type)][:in entity-id-field@@ -171,40 +159,48 @@:all nil)]}]])))))deps.dependency-types/dependency-type->model))-(defn- broken-entities-expr+(mu/defn- broken-entities-expr :- ::h2x/honeysql-expr"Matches entities at `entity-type-field`/`entity-id-field` that have failed analysis (an AnalysisFinding with`:result` false)."- [entity-type-field entity-id-field]+ [entity-type-field :- Column+ entity-id-field :- Column](into [:or](keep (fn [[entity-type _model]][:and[:= entity-type-field (name entity-type)][:in entity-id-field^:allow-subquery- {:select [:analyzed_entity_id]+ {:select [:entity_id]:from [:analysis_finding]:where [:and- [:= :analysis_finding.analyzed_entity_type (name entity-type)]+ [:= :analysis_finding.entity_type (name entity-type)][:= :analysis_finding.result false]]}]]))deps.dependency-types/dependency-type->model))-(defn- allowed-entity-types-expr+(mu/defn- allowed-entity-types-expr :- ::h2x/honeysql-expr"Matches entities at `entity-type-field` whose type is one of `entity-types`."- [entity-type-field entity-types]+ [entity-type-field :- Column+ entity-types :- [:set ::dependencies.schema/entity-type]](into [:or] (map (fn [entity-type] [:= entity-type-field (name entity-type)])) entity-types))-(defn- entity-ids-expr+(mu/defn- entity-ids-expr :- ::h2x/honeysql-expr"Matches entities at `entity-type-field`/`entity-id-field` that are of type `entity-type` and whose id is in`ids`."- [entity-type-field entity-id-field {:keys [entity-type ids]}]+ [entity-type-field :- Column+ entity-id-field :- Column+ {:keys [entity-type ids]} :- [:map {:closed true}+ [:entity-type ::dependencies.schema/entity-type]+ [:ids [:sequential ::deps.dependency-types/entity-id]]]][:and [:= entity-type-field (name entity-type)] [:in entity-id-field ids]])-(defn- edge-restriction-expr+(mu/defn- edge-restriction-expr :- [:maybe ::h2x/honeysql-expr]"The combined `:where` fragment for `restriction-spec` (nil for no filter), which may include `:visible` (opts for[[visible-entities-expr]]), `:broken?` (restrict to [[broken-entities-expr]]), `:types` (restrict to[[allowed-entity-types-expr]]), or `:entity-type` + `:ids` (restrict to [[entity-ids-expr]]). Combinablefragments are AND-ed together."- [entity-type-field entity-id-field {:keys [visible broken? types entity-type ids]}]+ [entity-type-field :- Column+ entity-id-field :- Column+ {:keys [visible broken? types entity-type ids]} :- ::dependencies.schema/edge-restriction](let [fragments (cond-> []visible (conj (visible-entities-expr entity-type-field entity-id-field visible))broken? (conj (broken-entities-expr entity-type-field entity-id-field))@@ -228,24 +224,27 @@[:src-id [:enum :from_entity_id :to_entity_id]][:dst-type [:enum :from_entity_type :to_entity_type]][:dst-id [:enum :from_entity_id :to_entity_id]]- [:entity-type EntityType]- [:entity-ids [:or [:set ::deps.dependency-types/entity-id] [:sequential ::deps.dependency-types/entity-id]]]- [:destination-restriction RestrictionSpec]- [:source-restriction RestrictionSpec]]]- (t2/select :model/Dependency- {:where (into [:and- [:= src-type (name entity-type)]- [:in src-id entity-ids]]- (keep identity)- [(edge-restriction-expr dst-type dst-id destination-restriction)- (edge-restriction-expr src-type src-id source-restriction)])}))+ [:entity-type ::dependencies.schema/entity-type]+ [:entity-ids [:sequential ::deps.dependency-types/entity-id]]+ [:destination-restriction ::dependencies.schema/edge-restriction]+ [:source-restriction ::dependencies.schema/edge-restriction]]]+ (into []+ (mapcat (fn [batch]+ (t2/select :model/Dependency+ {:where (into [:and+ [:= src-type (name entity-type)]+ [:in src-id batch]]+ (keep identity)+ [(edge-restriction-expr dst-type dst-id destination-restriction)+ (edge-restriction-expr src-type src-id source-restriction)])})))+ (partition-all 1000 entity-ids)))(mu/defn finding-errors-from-sources"The AnalysisFindingErrors caused by any of the entities `source-entity-type` `source-entity-ids`, whose analyzedentity is visible to the user described by `user-id`, `is-superuser?`, and `is-data-analyst?`."- [source-entity-type :- EntityType+ [source-entity-type :- ::dependencies.schema/entity-typesource-entity-ids :- [:sequential ::deps.dependency-types/entity-id]- {:keys [user-id is-superuser? is-data-analyst?]} :- VisibleOpts]+ {:keys [user-id is-superuser? is-data-analyst?]} :- ::dependencies.schema/visible-opts](t2/select :model/AnalysisFindingError{:where [:and[:= :source_entity_type (name source-entity-type)]@@ -257,9 +256,9 @@(mu/defn finding-errors-for-entities-with-visible-sources"The AnalysisFindingErrors analyzing any of the entities `entity-type` `entity-ids`, excluding those whose sourceentity exists but is not visible to the user described by `user-id`, `is-superuser?`, and `is-data-analyst?`."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-ids :- [:sequential ::deps.dependency-types/entity-id]- {:keys [user-id is-superuser? is-data-analyst?]} :- VisibleOpts]+ {:keys [user-id is-superuser? is-data-analyst?]} :- ::dependencies.schema/visible-opts](t2/select :model/AnalysisFindingError{:where [:and[:= :analyzed_entity_type (name entity-type)]@@ -275,21 +274,48 @@;;; `entity_type`/`entity_id`/`sort_key` rows, then page and count that union. These private helpers build the;;; per-entity-type SELECT and the pieces (joins, extra filters, sort expression) that make it up.-(defn- personal-root-collection-ids+(def ^:private DependencyItemParams+ "Params shared by [[dependency-item-ids]] and [[dependency-item-count]]."+ [:map {:closed true}+ [:query-type [:enum :unreferenced :breaking]]+ [:entity-types [:sequential ::deps.dependency-types/dependency-types]]+ [:card-types {:optional true} [:maybe [:sequential [:or :keyword :string]]]]+ [:search-text {:optional true} [:maybe :string]]+ [:include-personal-collections? :boolean]+ [:sort-column {:optional true} [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]]+ [:sort-direction [:enum :asc :desc]]+ [:offset ms/IntGreaterThanOrEqualToZero]+ [:limit ms/PositiveInt]+ [:user-id ::lib.schema.id/user]+ [:is-superuser? {:optional true} [:maybe :boolean]]+ [:is-data-analyst? {:optional true} [:maybe :boolean]]])++(def ^:private DependencyItemSelectParams+ "The [[DependencyItemParams]] of the SELECT of one `:entity-type`."+ (mut/assoc DependencyItemParams :entity-type ::deps.dependency-types/dependency-types))++(mu/defn- personal-root-collection-ids :- [:sequential ::lib.schema.id/collection]"The IDs of the personal Collections."[]- (t2/select-pks-vec :model/Collection :personal_owner_id [:not= nil] :location "/"))+ (or (t2/select-pks-vec :model/Collection :personal_owner_id [:not= nil] :location "/") []))-(defn action-root-namespace+(mu/defn action-root-namespace :- [:maybe [:= :data-actions]]"The namespace of the root collection an action without a collection sits in: the data actions root for a`model-less?` action, and the default root for an action of a model."- [model-less?]+ [model-less? :- :boolean](when model-less?:data-actions))-(defn- entity-type-config+(def ^:private EntityTypeConfig+ "What [[entity-type-config]] returns for an entity type."+ [:map {:closed true}+ [:table-name [:or :keyword ::h2x/honeysql-clause-opts]]+ [:name-column ::h2x/honeysql-expr]+ [:location-column ::h2x/honeysql-expr]])++(mu/defn- entity-type-config :- EntityTypeConfig"The table name and name/location column expressions to select an item list row for `entity-type`."- [entity-type]+ [entity-type :- ::deps.dependency-types/dependency-types](let [root-collection (collection.root/root-collection-with-ui-details(case entity-type:transform :transforms@@ -327,11 +353,15 @@:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)](:segment :measure) :table.display_name)}))-(defn- item-query-join+(mu/defn- item-query-join :- [:map {:closed true}+ [:join [:sequential HoneySQLForm]]+ [:join-filter ::h2x/honeysql-expr]]"The join and join-filter for `query-type`, restricting the joined-in dependency/finding-error rows to thosewhose other-side entity passes `default-visible-restriction` (see [[edge-restriction-expr]]; this is independent of anyarchived-items override applied to the item's own visibility)."- [query-type entity-type default-visible-restriction]+ [query-type :- [:enum :unreferenced :broken :breaking]+ entity-type :- ::deps.dependency-types/dependency-types+ default-visible-restriction :- ::dependencies.schema/edge-restriction](case query-type:unreferenced {:join [:dependency [:and[:= :dependency.to_entity_id :entity.id]@@ -340,8 +370,8 @@default-visible-restriction)]]:join-filter [:= :dependency.id nil]}:broken {:join [:analysis_finding [:and- [:= :analysis_finding.analyzed_entity_id :entity.id]- [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]+ [:= :analysis_finding.entity_id :entity.id]+ [:= :analysis_finding.entity_type (name entity-type)]]]:join-filter [:= :analysis_finding.result false]}:breaking {:join [:analysis_finding_error [:and[:= :analysis_finding_error.source_entity_id :entity.id]@@ -351,9 +381,9 @@default-visible-restriction)]]:join-filter [:!= :analysis_finding_error.id nil]}))-(defn- location-joins-for-entity+(mu/defn- location-joins-for-entity :- [:set Join]"The set of join keywords needed for location-based operations on `entity-type`."- [entity-type]+ [entity-type :- ::deps.dependency-types/dependency-types](case entity-type:card #{:collection :dashboard :document}(:transform :snippet :dashboard :document :action) #{:collection}@@ -361,11 +391,13 @@:table #{:database}#{}))-(defn- optional-item-expr-and-joins+(mu/defn- optional-item-expr-and-joins :- [:map {:closed true}+ [:exprs [:sequential ::h2x/honeysql-expr]]+ [:joins [:set Join]]]"The extra `:where` fragments and joins for `params`' card-type, search-text, database, internal-content,archived, and personal-collection restrictions."- [{:keys [query-type entity-type card-types search-text include-personal-collections?]}- {:keys [name-column location-column]}]+ [{:keys [query-type entity-type card-types search-text include-personal-collections?]} :- DependencyItemSelectParams+ {:keys [name-column location-column]} :- EntityTypeConfig](let [card-type-expr (when (and (= entity-type :card) (seq card-types)){:expr [:in :entity.type (mapv name card-types)]:joins #{}})@@ -416,10 +448,16 @@{:exprs (keep :expr results):joins (reduce set/union #{} (map :joins results))}))-(defn- sort-key-expr-and-joins+(mu/defn- sort-key-expr-and-joins :- [:map {:closed true}+ [:sort-expr HoneySQLForm]+ [:joins [:set Join]]]"The sort expression and extra joins for `sort-column`, restricting the `dependents-errors`/`dependents-with-errors`counts to finding errors whose analyzed entity passes `default-visible-restriction` (see [[edge-restriction-expr]])."- [sort-column entity-type name-column location-column default-visible-restriction]+ [sort-column :- [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]+ entity-type :- ::deps.dependency-types/dependency-types+ name-column :- ::h2x/honeysql-expr+ location-column :- ::h2x/honeysql-expr+ default-visible-restriction :- ::dependencies.schema/edge-restriction](case sort-column:location {:sort-expr location-column:joins (location-joins-for-entity entity-type)}@@ -446,10 +484,11 @@{:sort-expr name-column:joins #{}}))-(defn- build-left-joins+(mu/defn- build-left-joins :- [:sequential HoneySQLForm]"`base-join` with the LEFT JOINs needed for `joins` (a set of `:database`, `:collection`, `:dashboard`,`:document`, and/or `:table`) appended."- [base-join joins]+ [base-join :- [:sequential HoneySQLForm]+ joins :- [:set Join]](cond-> base-join(:database joins) (conj [:metabase_database :database] [:= :entity.db_id :database.id])(:collection joins) (conj :collection [:= :entity.collection_id :collection.id])@@ -457,13 +496,14 @@(:document joins) (conj :document [:= :entity.document_id :document.id])(:table joins) (conj (warehouse-schema-overlay/table-query {:alias :table}) [:= :entity.table_id :table.id])))-(defn- dependency-item-select+(mu/defn- dependency-item-select :- ::h2x/honeysql-clause-opts"The per-entity-type SELECT that [[dependency-item-ids]] and [[dependency-item-count]] union together, matching`query-type` (`:unreferenced` or `:breaking`) items of `entity-type`, restricted to `card-types`, `search-text`,and `include-personal-collections?`, visible to the user described by `user-id`, `is-superuser?`, and`is-data-analyst?`, with `sort-column`'s expression selected as `:sort_key`. Throws when `user-id` is missing,since the visibility restriction must always be applied."- [{:keys [query-type entity-type sort-column user-id is-superuser? is-data-analyst?] :as params}]+ [{:keys [query-type entity-type sort-column user-id is-superuser? is-data-analyst?] :as params}+ :- DependencyItemSelectParams](when-not user-id(throw (ex-info "dependency-item-select requires a user-id so the visibility restriction is always applied"{:query-type query-type :entity-type entity-type})))@@ -489,22 +529,6 @@:left-join (build-left-joins join all-joins):where (into [:and join-filter item-visible-expr] (keep identity) exprs)}))-(def ^:private DependencyItemParams- "Params shared by [[dependency-item-ids]] and [[dependency-item-count]]."- [:map {:closed true}- [:query-type [:enum :unreferenced :breaking]]- [:entity-types [:sequential ::deps.dependency-types/dependency-types]]- [:card-types {:optional true} [:maybe [:sequential [:or :keyword :string]]]]- [:search-text {:optional true} [:maybe :string]]- [:include-personal-collections? :boolean]- [:sort-column {:optional true} [:maybe [:enum :name :location :dependents-errors :dependents-with-errors]]]- [:sort-direction [:enum :asc :desc]]- [:offset ms/IntGreaterThanOrEqualToZero]- [:limit ms/PositiveInt]- [:user-id ::lib.schema.id/user]- [:is-superuser? {:optional true} [:maybe :boolean]]- [:is-data-analyst? {:optional true} [:maybe :boolean]]])-(mu/defn dependency-item-ids"A page of `[entity-type entity-id]` pairs for `query-type` (`:unreferenced` or `:breaking`), restricted to`entity-types`, `card-types` (applied only to `:card` entities), `search-text` (nil for none), and@@ -538,7 +562,7 @@[{:keys [source-entity-type source-entity-id dependent-types dependent-card-typesuser-id is-superuser? is-data-analyst?]}:- [:map {:closed true}- [:source-entity-type EntityType]+ [:source-entity-type ::dependencies.schema/entity-type][:source-entity-id ms/PositiveInt][:dependent-types {:optional true} [:maybe [:sequential :string]]][:dependent-card-types {:optional true} [:maybe [:sequential :string]]]@@ -550,8 +574,8 @@:from [[:analysis_finding_error :afe]]:join [[:analysis_finding :af][:and- [:= :af.analyzed_entity_type :afe.analyzed_entity_type]- [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]+ [:= :af.entity_type :afe.analyzed_entity_type]+ [:= :af.entity_id :afe.analyzed_entity_id]]]:where (cond-> [:and[:= :afe.source_entity_type (name source-entity-type)][:= :afe.source_entity_id source-entity-id]@@ -571,29 +595,30 @@;;; ---------------------------------------------------- Entities -----------------------------------------------------(mu/defn instances- "The instances of the entity type `entity-type` with `ids`."+(mu/defn entities+ "The entities of the entity type `entity-type` with `ids`."[entity-type :- ::deps.dependency-types/dependency-typesids :- [:set ::deps.dependency-types/entity-id]](t2/select (deps.dependency-types/dependency-type->model entity-type) :id [:in ids]))-(defn- entity-source+(mu/defn- entity-source :- [:map {:closed true}+ [:from {:optional true} [:sequential HoneySQLForm]]]"What a read of `entity-type` selects from: the overlay for a Table, else the model's own table."- [entity-type]+ [entity-type :- ::deps.dependency-types/dependency-types](if (= entity-type :table){:from [(warehouse-schema-overlay/table-query)]}{}))-(mu/defn instances-with-columns- "The `columns` of the instances of the entity type `entity-type` with `ids`."+(mu/defn entities-with-columns+ "The `columns` of the entities of the entity type `entity-type` with `ids`."[entity-type :- ::deps.dependency-types/dependency-typescolumns :- [:sequential :keyword]ids :- [:sequential ::deps.dependency-types/entity-id]](t2/select (into [(deps.dependency-types/dependency-type->model entity-type)] columns):id [:in ids] (entity-source entity-type)))-(mu/defn instance-with-columns- "The `columns` of the instance of the entity type `entity-type` with `id`, or nil."+(mu/defn entity-with-columns+ "The `columns` of the entity of the entity type `entity-type` with `id`, or nil."[entity-type :- ::deps.dependency-types/dependency-typescolumns :- [:sequential :keyword]id :- ::deps.dependency-types/entity-id]@@ -645,16 +670,16 @@[transform-ids :- [:sequential ::lib.schema.id/transform]](t2/select [:model/Transform :id :source] :id [:in transform-ids]))-(mu/defn transform-ids-of-source-database+(mu/defn source-database-transform-ids :- [:set ::lib.schema.id/transform]"The IDs of the Transforms reading from the Database with `db-id`."[db-id :- ::lib.schema.id/database]- (t2/select-pks-set :model/Transform :source_database_id db-id))+ (or (t2/select-pks-set :model/Transform :source_database_id db-id) #{}));;; -------------------------------------------------- Dependencies --------------------------------------------------(mu/defn dependencies-from"The `:id`, `:to_entity_type`, and `:to_entity_id` of the Dependencies of the entity `entity-type` `entity-id`."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-id :- ms/PositiveInt](t2/select [:model/Dependency :id :to_entity_type :to_entity_id]:from_entity_type entity-type@@ -662,9 +687,9 @@(mu/defn dependency-exists?"Whether the entity `from-type` `from-id` depends on the entity `to-type` `to-id`."- [from-type :- EntityType+ [from-type :- ::dependencies.schema/entity-typefrom-id :- ms/PositiveInt- to-type :- EntityType+ to-type :- ::dependencies.schema/entity-typeto-id :- ms/PositiveInt](t2/exists? :model/Dependency:from_entity_type from-type :from_entity_id from-id@@ -683,11 +708,11 @@(mu/defn retarget-dependency!"Point the Dependency of the entity `from-type` `from-id` on the entity `old-to-type` `old-to-id` at the entity`new-to-type` `new-to-id`, returning the number updated."- [from-type :- EntityType+ [from-type :- ::dependencies.schema/entity-typefrom-id :- ms/PositiveInt- old-to-type :- EntityType+ old-to-type :- ::dependencies.schema/entity-typeold-to-id :- ms/PositiveInt- new-to-type :- EntityType+ new-to-type :- ::dependencies.schema/entity-typenew-to-id :- ms/PositiveInt](t2/update! :model/Dependency{:from_entity_type from-type :from_entity_id from-id@@ -702,9 +727,9 @@(mu/defn delete-dependency!"Delete the Dependency of the entity `from-type` `from-id` on the entity `to-type` `to-id`, returning the numberdeleted."- [from-type :- EntityType+ [from-type :- ::dependencies.schema/entity-typefrom-id :- ms/PositiveInt- to-type :- EntityType+ to-type :- ::dependencies.schema/entity-typeto-id :- ms/PositiveInt](t2/delete! :model/Dependency:from_entity_type from-type :from_entity_id from-id@@ -712,7 +737,7 @@(mu/defn delete-dependencies-from!"Delete the Dependencies of the entities `entity-type` `entity-ids`, returning the number deleted."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-ids :- [:sequential ms/PositiveInt]](t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id [:in entity-ids]))@@ -735,250 +760,360 @@:to_entity_type :transform:to_entity_id transform-id))-;;; ------------------------------------------------ Dependency status --------------------------------------------------(def ^:private entity-database-queries- "For each entity type that can hold a native query, a query of the IDs of its entities reading from `db-id`."- {:card (fn [db-id] ^:allow-subquery {:select [:id]- :from [(t2/table-name :model/Card)]- :where [:= :database_id db-id]})- :transform (fn [db-id] ^:allow-subquery {:select [:id]- :from [(t2/table-name :model/Transform)]- :where [:= :source_database_id db-id]})- :action (fn [db-id] ^:allow-subquery {:select [:action_id]- :from [(t2/table-name :model/QueryAction)]- :where [:= :database_id db-id]})})--(mu/defn unresolved-table-entity-ids :- [:map-of ::deps.dependency-types/dependency-types [:set ms/PositiveInt]]- "The IDs, by entity type, of the entities reading from the Database with `db-id` that name tables matching nothing."- [db-id :- ::lib.schema.id/database]- (into {}- (keep (fn [[entity-type entity-ids-query]]- (when-let [ids (not-empty (t2/select-fn-set :entity_id :model/DependencyStatus- :entity_type entity-type- :unresolved_tables true- :entity_id [:in (entity-ids-query db-id)]))]- [entity-type ids])))- entity-database-queries))--(mu/defn dependency-status- "The DependencyStatus of the entity `entity-type` `entity-id`, or nil."- [entity-type :- EntityType- entity-id :- ms/PositiveInt]- (t2/select-one :model/DependencyStatus :entity_type entity-type :entity_id entity-id))+;;; -------------------------------------------- Statuses, shared by the jobs --------------------------------------------++(def ^:private StatusModel+ "A job's status model."+ [:enum :model/DependencyStatus :model/ResultMetadataStatus :model/AnalysisFindingStatus])++(def ^:private one-more-write+ "The `stale` count of a status after one more write changed what its job reads, as Honey SQL."+ [:+ :stale [:inline 1]])++(def ^:private fresh-attempts+ "The columns of a status a write resets, so that the job tries an entity it gave up on again."+ {:fail_count 0 :terminal false})++(mu/defn- mark-stale!+ "Count one more write in the statuses in the status `model` of the entities `type->ids`, with fresh attempts, so the+ job processes them until it clears a count no write raised meanwhile; an entity without a status is processed+ anyway."+ [model :- StatusModel+ type->ids :- ::dependencies.schema/type->ids]+ (doseq [[entity-type ids] type->ids+ batch (partition-all 1000 ids)]+ (t2/update! model :entity_type entity-type :entity_id [:in batch] (assoc fresh-attempts :stale one-more-write))))++(mu/defn- mark-dependents-stale!+ "Count one more write in the statuses in the status `model` of the entities directly depending on the entities+ `entity-type` `entity-ids`, in one UPDATE per 1,000 of them."+ [model :- StatusModel+ entity-type :- ::dependencies.schema/entity-type+ entity-ids :- [:set ms/PositiveInt]]+ (let [table (t2/table-name model)]+ (doseq [batch (partition-all 1000 entity-ids)]+ (t2/query {:update table+ :set (assoc fresh-attempts :stale one-more-write)+ :where [:exists ^:allow-subquery+ {:select [1]+ :from [[(t2/table-name :model/Dependency) :d]]+ :where [:and+ [:= :d.to_entity_type (name entity-type)]+ [:in :d.to_entity_id batch]+ [:= :d.from_entity_type (u/qualified-key table :entity_type)]+ [:= :d.from_entity_id (u/qualified-key table :entity_id)]]}]}))))++(mu/defn- clear-stale! :- :boolean+ "Record that a job processed the entity `entity-type` `entity-id` with its logic of `current-version`, clearing the+ `stale` count of its status in the status `model` when it is still the `stale` the job picked it with, so a write+ while it was processed has the job process it again, returning whether it cleared it."+ [model :- StatusModel+ entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ stale :- ms/IntGreaterThanOrEqualToZero+ current-version :- ms/PositiveInt]+ (pos? (t2/update! model :entity_type entity-type :entity_id entity-id :stale stale+ {:stale 0 :fail_count 0 :analysis_version current-version})))++(mu/defn- record-failure! :- :boolean+ "Record that a job failed to process the entity `entity-type` `entity-id`, giving up on it until a write changes it+ once it failed `max-failures` times in a row, returning whether it gave up."+ [model :- StatusModel+ entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ max-failures :- ms/PositiveInt]+ (t2/update! model :entity_type entity-type :entity_id entity-id {:fail_count [:+ :fail_count [:inline 1]]})+ (pos? (t2/update! model :entity_type entity-type :entity_id entity-id :fail_count [:>= max-failures]+ {:terminal true})))++(mu/defn mark-dependency-statuses-stale!+ "Mark the DependencyStatuses of the entities `type->ids` stale, so the dependency job computes their dependencies+ again."+ [type->ids :- ::dependencies.schema/type->ids]+ (mark-stale! :model/DependencyStatus type->ids))++(mu/defn mark-result-metadata-statuses-stale!+ "Mark the ResultMetadataStatuses of the entities `type->ids` stale, so the result metadata job re-derives them."+ [type->ids :- ::dependencies.schema/type->ids]+ (mark-stale! :model/ResultMetadataStatus type->ids))++(mu/defn mark-analysis-finding-statuses-stale!+ "Mark the AnalysisFindingStatuses of the entities `type->ids` stale, so the analysis job analyzes them again."+ [type->ids :- ::dependencies.schema/type->ids]+ (mark-stale! :model/AnalysisFindingStatus type->ids))++(mu/defn mark-dependents-result-metadata-statuses-stale!+ "Mark the ResultMetadataStatuses of the entities directly depending on the entities `entity-type` `entity-ids` stale."+ [entity-type :- ::dependencies.schema/entity-type+ entity-ids :- [:set ms/PositiveInt]]+ (mark-dependents-stale! :model/ResultMetadataStatus entity-type entity-ids))++(mu/defn mark-dependents-analysis-finding-statuses-stale!+ "Mark the AnalysisFindingStatuses of the entities directly depending on the entities `entity-type` `entity-ids`+ stale."+ [entity-type :- ::dependencies.schema/entity-type+ entity-ids :- [:set ms/PositiveInt]]+ (mark-dependents-stale! :model/AnalysisFindingStatus entity-type entity-ids))++(mu/defn- current-dependencies-clause+ "Whether the dependency status of the entity of `entity-type` with `id-field` is current for the dependency logic of+ `dependency-version`, as Honey SQL."+ [entity-type :- ::deps.dependency-types/dependency-types+ id-field :- :keyword+ dependency-version :- ms/PositiveInt]+ [:exists ^:allow-subquery {:select [1]+ :from [[:dependency_status :ds]]+ :where [:and+ [:= :ds.entity_type (name entity-type)]+ [:= :ds.entity_id id-field]+ [:= :ds.stale 0]+ [:= :ds.analysis_version dependency-version]]}])++(mu/defn- pending-statuses :- [:sequential ::dependencies.schema/pending-status]+ "Up to `batch-size` entities of `entity-type` without a status in `status-model`, or whose status the job has not+ given up on and that counts a write or records a version of the job's logic below `current-version`, those last,+ with their `stale` counts, nil without a status; `user-mbql-only?` narrows cards to MBQL ones that Metabase did not create itself, and `dependency-version`+ to the entities whose dependencies are current."+ [entity-type :- ::deps.dependency-types/dependency-types+ status-model :- StatusModel+ batch-size :- ms/PositiveInt+ current-version :- ms/PositiveInt+ {:keys [user-mbql-only? dependency-version]} :- [:map {:closed true}+ [:user-mbql-only? {:optional true} :boolean]+ [:dependency-version {:optional true} ms/PositiveInt]]]+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))+ id-field (u/qualified-key table-name :id)+ outdated [:< :status.analysis_version current-version]]+ (mapv #(select-keys % [:entity_id :stale])+ (t2/query+ {:select [[id-field :entity_id] [:status.stale :stale]]+ :from [table-name]+ :left-join [[(t2/table-name status-model) :status] [:and+ [:= :status.entity_type (name entity-type)]+ [:= :status.entity_id id-field]]]+ :where (cond-> [:and [:or [:= :status.id nil]+ [:and [:= :status.terminal false] [:or [:> :status.stale 0] outdated]]]]+ user-mbql-only? (conj [:or [:= :report_card.query_type nil]+ [:not= :report_card.query_type "native"]]+ (mi/exclude-internal-content-hsql :model/Card+ :table-alias :report_card))+ dependency-version (conj (current-dependencies-clause+ entity-type id-field dependency-version)))+ :order-by [[[:case outdated [:inline 1] :else [:inline 0]]]]+ :limit batch-size}))))++(mu/defn- take-pending-statuses! :- [:sequential ::dependencies.schema/pending-status]+ "The [[pending-statuses]] a job is to process, each with the `stale` count it clears afterwards, inserting a status+ counting one write for each entity without one before the job reads it, so a write while it is processed counts."+ [entity-type :- ::deps.dependency-types/dependency-types+ status-model :- StatusModel+ batch-size :- ms/PositiveInt+ current-version :- ms/PositiveInt+ opts :- [:map {:closed true}+ [:user-mbql-only? {:optional true} :boolean]+ [:dependency-version {:optional true} ms/PositiveInt]]]+ (let [statuses (pending-statuses entity-type status-model batch-size current-version opts)+ missing (remove :stale statuses)]+ (when (seq missing)+ (t2/insert! status-model (for [{:keys [entity_id]} missing]+ {:entity_type entity-type+ :entity_id entity_id+ :stale 1+ :analysis_version 0})))+ (mapv #(update % :stale (fnil identity 1)) statuses)))++(mu/defn- deleted-entity-ids-with-status :- [:sequential ms/PositiveInt]+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have a status in `status-model`."+ [status-model :- StatusModel+ entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]+ (mapv :entity_id (t2/query {:select [:status.entity_id]+ :from [[(t2/table-name status-model) :status]]+ :where [:and+ [:= :status.entity_type (name entity-type)]+ [:not [:exists ^:allow-subquery {:select [1]+ :from [[table-name :entity]]+ :where [:= :entity.id :status.entity_id]}]]]+ :limit batch-size}))))++;;; ---------------------------------------------- Dependency job ----------------------------------------------++(mu/defn dependency-deleted-entity-ids :- [:sequential ms/PositiveInt]+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have a DependencyStatus."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (deleted-entity-ids-with-status :model/DependencyStatus entity-type batch-size))(mu/defn delete-dependency-statuses!"Delete the DependencyStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-ids :- [:sequential ms/PositiveInt]](t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id [:in entity-ids]))-(mu/defn deleted-entity-ids :- [:sequential ms/PositiveInt]- "Up to `batch-size` IDs of deleted entities of the type `entity-type` that still have a DependencyStatus."- [entity-type :- ::deps.dependency-types/dependency-types- batch-size :- ms/PositiveInt]- (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]- (vec (t2/select-fn-vec :entity_id :model/DependencyStatus- {:where [:and- [:= :dependency_status.entity_type (name entity-type)]- [:not [:exists ^:allow-subquery- {:select [1]- :from [[table-name :entity]]- :where [:= :entity.id :dependency_status.entity_id]}]]]- :limit batch-size}))))--(mu/defn mark-dependency-status-stale!- "Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it- doesn't exist. Resets retry state so previously-failed entities get a fresh chance."- [entity-type :- EntityType- entity-id :- ms/PositiveInt]- (mdb/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id entity-id}- (fn [existing]- (if existing- {:stale true :fail_count 0 :next_retry_at nil :terminal false}- {:stale true :dependency_analysis_version 0}))))--(mu/defn upsert-dependency-status!- "Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to- `current-version`, `unresolved_tables` to `unresolved-tables?`, and clearing any failure state."- [entity-type :- EntityType- entity-id :- ms/PositiveInt- current-version :- ms/PositiveInt- unresolved-tables? :- :boolean]- (mdb/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id entity-id}- (fn [_existing]- {:dependency_analysis_version current-version- :stale false- :fail_count 0- :next_retry_at nil- :unresolved_tables unresolved-tables?})))--(mu/defn record-dependency-status-failure!- "Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives the- existing DependencyStatus row (or nil if none exists) and must return the columns to set."- [entity-type :- EntityType- entity-id :- ms/PositiveInt- update-fn :- fn?]- (mdb/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id entity-id}- update-fn))--(mu/defn pending-retry-exists?- "Whether a non-terminal DependencyStatus is waiting for a retry."- []- (t2/exists? :model/DependencyStatus :terminal false :next_retry_at [:not= nil]))+(mu/defn record-dependency-failure! :- :boolean+ "Record that the dependency job failed to compute the dependencies of the entity `entity-type` `entity-id`, giving up on it until a write changes it once it failed+ `max-failures` times in a row, returning whether it gave up."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ max-failures :- ms/PositiveInt]+ (record-failure! :model/DependencyStatus entity-type entity-id max-failures))++(mu/defn clear-dependency-status! :- :boolean+ "Record that the dependency job computed the dependencies of the entity `entity-type` `entity-id` with the analysis+ of `current-version`, clearing its count of `stale` writes unless a write raised it, returning whether it cleared it."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ stale :- ms/IntGreaterThanOrEqualToZero+ current-version :- ms/PositiveInt]+ (clear-stale! :model/DependencyStatus entity-type entity-id stale current-version))++(mu/defn dependency-pending? :- :boolean+ "Whether an entity of the type `entity-type` has no DependencyStatus, or one that counts a write or records a version+ of the dependency logic below `current-version`."+ [entity-type :- ::deps.dependency-types/dependency-types+ current-version :- ms/PositiveInt]+ (boolean (seq (pending-statuses entity-type :model/DependencyStatus 1 current-version {}))))-(mu/defn instances-for-dependency-calculation- "Up to `batch-size` instances of the entity type `entity-type` without a DependencyStatus, or whose status is- stale or below `current-version`, is not terminal, and whose retry time has passed at `now`; stale ones first."+(mu/defn take-pending-dependency-statuses! :- [:sequential ::dependencies.schema/pending-status]+ "Up to `batch-size` entities of the type `entity-type` without a DependencyStatus, or whose status counts a write or+ records a version of the dependency logic below `current-version`, outdated ones last, with the `stale` count the job+ clears afterwards."[entity-type :- ::deps.dependency-types/dependency-typesbatch-size :- ms/PositiveInt- current-version :- ms/PositiveInt- now :- ms/TemporalInstant]- (let [model (deps.dependency-types/dependency-type->model entity-type)- table-name (t2/table-name model)- id-field (keyword (name table-name) "id")- table-wildcard (keyword (name table-name) "*")]- (t2/select model- {:select [table-wildcard]- :from table-name- :left-join [:dependency_status [:and- [:= :dependency_status.entity_id id-field]- [:= :dependency_status.entity_type (name entity-type)]]]- :where [:or- [:= :dependency_status.entity_id nil]- [:and- [:or- [:= :dependency_status.stale true]- [:< :dependency_status.dependency_analysis_version current-version]]- [:= :dependency_status.terminal false]- [:or- [:is :dependency_status.next_retry_at nil]- [:<= :dependency_status.next_retry_at now]]]]- :order-by [[[:case- [:or [:= :dependency_status.entity_id nil] [:= :dependency_status.stale true]] [:inline 0]- :else [:inline 1]]]]- :limit batch-size})))--;;; ------------------------------------------------ Analysis findings --------------------------------------------------(mu/defn finding-id- "The ID of the AnalysisFinding of the entity `entity-type` `entity-id`, or nil."- [entity-type :- EntityType- entity-id :- ms/PositiveInt]- (t2/select-one-fn :id [:model/AnalysisFinding :id]- :analyzed_entity_type entity-type- :analyzed_entity_id entity-id))--(mu/defn insert-finding!- "Insert the AnalysisFinding `row`, returning the number inserted."- [row :- [:map {:closed true}- [:id {:optional true} ms/PositiveInt]- [:analyzed_at {:optional true} [:maybe ms/TemporalInstant]]- [:analysis_version {:optional true} ms/PositiveInt]- [:result {:optional true} :boolean]- [:stale {:optional true} :boolean]- [:analyzed_entity_type {:optional true} [:maybe [:or :keyword :string]]]- [:analyzed_entity_id {:optional true} [:maybe ms/PositiveInt]]]]- (t2/insert! :model/AnalysisFinding row))--(mu/defn update-finding!- "Apply `changes` to the AnalysisFinding with `finding-id`, returning the number updated."- [finding-id :- ms/PositiveInt- changes :- (mut/select-keys ::dependencies.schema/analysis-finding.update [:analyzed_at :analysis_version :result :stale])]- (t2/update! :model/AnalysisFinding finding-id changes))--(mu/defn mark-findings-stale!- "Mark the AnalysisFindings of the entities `entity-type` `entity-ids` as stale, returning the number updated."- [entity-type :- EntityType- entity-ids :- [:sequential ms/PositiveInt]]- (t2/update! :model/AnalysisFinding- :analyzed_entity_type entity-type- :analyzed_entity_id [:in entity-ids]- {:stale true}))+ current-version :- ms/PositiveInt]+ (take-pending-statuses! entity-type :model/DependencyStatus batch-size current-version {}))-(mu/defn stale-finding-exists?- "Whether a stale AnalysisFinding exists."- []- (t2/exists? :model/AnalysisFinding :stale true))+;;; ------------------------------------------- Result metadata job --------------------------------------------(mu/defn stale-finding-count- "The number of stale AnalysisFindings."- []- (t2/count :model/AnalysisFinding :stale true))+(mu/defn take-pending-result-metadata-statuses! :- [:sequential ::dependencies.schema/pending-status]+ "Up to `batch-size` MBQL cards Metabase did not create itself without a ResultMetadataStatus, or whose status counts a+ write or records a version of the logic below `current-version`, and whose dependencies the dependency logic of+ `dependency-version` computed, outdated ones last, with the `stale` count the job clears afterwards."+ [batch-size :- ms/PositiveInt+ current-version :- ms/PositiveInt+ dependency-version :- ms/PositiveInt]+ (take-pending-statuses! :card :model/ResultMetadataStatus batch-size current-version+ {:user-mbql-only? true+ :dependency-version dependency-version}))++(mu/defn record-result-metadata-failure! :- :boolean+ "Record that the result metadata job failed to re-derive the Card with `card-id`, giving up on it until a write+ changes it once it failed `max-failures` times in a row, returning whether it gave up."+ [card-id :- ::lib.schema.id/card+ max-failures :- ms/PositiveInt]+ (record-failure! :model/ResultMetadataStatus :card card-id max-failures))++(mu/defn clear-result-metadata-status! :- :boolean+ "Record that the result metadata job re-derived the Card with `card-id` with the logic of `current-version`, clearing+ its count of `stale` writes unless a write raised it, returning whether it cleared it."+ [card-id :- ::lib.schema.id/card+ stale :- ms/IntGreaterThanOrEqualToZero+ current-version :- ms/PositiveInt]+ (clear-stale! :model/ResultMetadataStatus :card card-id stale current-version))-(mu/defn instances-for-analysis- "Up to `batch-size` instances of the entity type `entity-type` whose AnalysisFinding is stale, missing, or below- `current-version`; stale ones first, then the longest unanalyzed."- [entity-type :- ::deps.dependency-types/dependency-types- batch-size :- ms/PositiveInt+(mu/defn result-metadata-deleted-card-ids :- [:sequential ::lib.schema.id/card]+ "Up to `batch-size` IDs of deleted Cards that still have a ResultMetadataStatus."+ [batch-size :- ms/PositiveInt]+ (deleted-entity-ids-with-status :model/ResultMetadataStatus :card batch-size))++(mu/defn delete-result-metadata-statuses!+ "Delete the ResultMetadataStatuses of the Cards with `card-ids`, returning the number deleted."+ [card-ids :- [:sequential ::lib.schema.id/card]]+ (t2/delete! :model/ResultMetadataStatus :entity_type :card :entity_id [:in card-ids]))++(mu/defn remapped-table-ids :- [:set ::lib.schema.id/table]+ "The IDs of the Tables of the Fields remapped to one of the Fields with `field-ids` as their human-readable field."+ [field-ids :- [:set ::lib.schema.id/field]]+ (or (t2/select-fn-set :table_id [:model/Field :metabase_field.table_id]+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]+ :join [:dimension [:= :dimension.field_id :metabase_field.id]]+ :where [:in :dimension.human_readable_field_id field-ids]})+ #{}))++(mu/defn internal-remap? :- :boolean+ "Whether the Field with `field-id` has an internal remap, which shows its FieldValues' human-readable values."+ [field-id :- ::lib.schema.id/field]+ (t2/exists? :model/Dimension :field_id field-id :type "internal"))++(mu/defn native-card-ids :- [:set ::lib.schema.id/card]+ "The IDs of the native Cards among `card-ids`."+ [card-ids :- [:set ::lib.schema.id/card]]+ (or (t2/select-pks-set :model/Card :id [:in card-ids] :query_type :native) #{}))++(mu/defn card-state :- [:maybe ::dependencies.schema/card-state]+ "The `updated_at` and `result_metadata` of the Card with `card-id`, locking its row until the transaction ends when+ `lock?`, or nil."+ [card-id :- ::lib.schema.id/card+ lock? :- :boolean]+ (some-> (t2/select-one [:model/Card :updated_at :result_metadata]+ (cond-> {:where [:= :id card-id]} lock? (assoc :for :update)))+ (select-keys [:updated_at :result_metadata])))++(mu/defn set-card-result-metadata!+ "Set the result metadata of the Card with `card-id` to `result-metadata` through the Card model, without touching+ `updated_at`."+ [card-id :- ::lib.schema.id/card+ result-metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]]+ (t2/update! :model/Card card-id {:result_metadata result-metadata+ :updated_at :updated_at}))++;;; ------------------------------------------- Analysis finding job -------------------------------------------++(mu/defn upsert-finding!+ "Insert or update the AnalysisFinding of the entity `entity-type` `entity-id` with `result`, updating the row a+ concurrent analysis inserted meanwhile."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ result :- :boolean]+ (mdb/update-or-insert! :model/AnalysisFinding {:entity_type entity-type :entity_id entity-id}+ (constantly {:result result})))++(mu/defn take-pending-analysis-finding-statuses! :- [:sequential ::dependencies.schema/pending-status]+ "Up to `batch-size` entities of the type `entity-type` without an AnalysisFindingStatus, or whose status counts a+ write or records an analysis below `current-version`, and whose dependencies the analysis of `dependency-version`+ computed, outdated ones last, with the `stale` count the job clears afterwards."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt+ current-version :- ms/PositiveInt+ dependency-version :- ms/PositiveInt]+ (take-pending-statuses! entity-type :model/AnalysisFindingStatus batch-size current-version+ {:dependency-version dependency-version}))++(mu/defn record-analysis-finding-failure! :- :boolean+ "Record that the analysis job failed to analyze the entity `entity-type` `entity-id`, giving up on it until a write changes it once it failed+ `max-failures` times in a row, returning whether it gave up."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ max-failures :- ms/PositiveInt]+ (record-failure! :model/AnalysisFindingStatus entity-type entity-id max-failures))++(mu/defn clear-analysis-finding-status! :- :boolean+ "Record that the analysis job analyzed the entity `entity-type` `entity-id` with the analysis of `current-version`,+ clearing its count of `stale` writes unless a write raised it, returning whether it cleared it."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ stale :- ms/IntGreaterThanOrEqualToZerocurrent-version :- ms/PositiveInt]- (let [model (deps.dependency-types/dependency-type->model entity-type)- table-name (t2/table-name model)- id-field (keyword (name table-name) "id")- table-wildcard (keyword (name table-name) "*")]- (t2/select model- {:select [table-wildcard]- :from table-name- :left-join [:analysis_finding [:and- [:= :analysis_finding.analyzed_entity_id id-field]- [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]- :where [:or- [:= :analysis_finding.stale true]- [:<- [:coalesce :analysis_finding.analysis_version 0]- current-version]]- :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]- [:analysis_finding.analyzed_at :asc]]- :limit batch-size})))--(mu/defn table-ids-with-outdated-findings- "The IDs of the Tables of the Database with `db-id` whose dependents were analyzed before the Table, one of its- Fields, or one of its Fields' user settings last changed."- [db-id :- ::lib.schema.id/database]- (t2/select-fn-set :table_id :model/AnalysisFinding- {:select [:field_updates/table_id]- :from [[^:allow-subquery {:select [[:table/id :table_id]- [:table/updated_at :last_table_update]- [[:max [:case- [:>= :field/updated_at- [:coalesce :field_settings/updated_at :field/updated_at]]- :field/updated_at- :else :field_settings/updated_at]]- :last_field_update]]- :from [(warehouse-schema-overlay/table-query {:alias :table})]- :left-join [(warehouse-schema-overlay/field-query {:alias :field})- [:= :field/table_id :table/id]- [(t2/table-name :model/FieldUserSettings) :field_settings]- [:= :field_settings/field_id :field/id]]- :where [:= :table/db_id db-id]- :group-by [:table/id- :table/updated_at]}- :field_updates]]- :inner-join [[(t2/table-name :model/Dependency) :dep]- [:and- [:= :dep/to_entity_type "table"]- [:= :field_updates/table_id :dep/to_entity_id]]- [(t2/table-name :model/AnalysisFinding) :finding]- [:and- [:= :finding/analyzed_entity_type :dep/from_entity_type]- [:= :finding/analyzed_entity_id :dep/from_entity_id]]]- :where [:and- [:!= :finding/analyzed_entity_id nil]- [:or- [:< :finding/analyzed_at :field_updates/last_table_update]- [:< :finding/analyzed_at :field_updates/last_field_update]]]}))+ (clear-stale! :model/AnalysisFindingStatus entity-type entity-id stale current-version))++(mu/defn analysis-finding-deleted-entity-ids :- [:sequential ms/PositiveInt]+ "Up to `batch-size` IDs of deleted entities of `entity-type` that still have an AnalysisFindingStatus."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (deleted-entity-ids-with-status :model/AnalysisFindingStatus entity-type batch-size))++(mu/defn delete-analysis-finding-statuses!+ "Delete the AnalysisFindingStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."+ [entity-type :- ::dependencies.schema/entity-type+ entity-ids :- [:sequential ms/PositiveInt]]+ (t2/delete! :model/AnalysisFindingStatus :entity_type entity-type :entity_id [:in entity-ids]));;; --------------------------------------------- Analysis finding errors ---------------------------------------------(mu/defn finding-errors-for-entity"The AnalysisFindingErrors of the entity `entity-type` `entity-id`."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-id :- ms/PositiveInt](t2/select :model/AnalysisFindingError :analyzed_entity_type entity-type :analyzed_entity_id entity-id))@@ -995,6 +1130,6 @@(mu/defn delete-finding-errors-for-entity!"Delete the AnalysisFindingErrors of the entity `entity-type` `entity-id`, returning the number deleted."- [entity-type :- EntityType+ [entity-type :- ::dependencies.schema/entity-typeentity-id :- ms/PositiveInt](t2/delete! :model/AnalysisFindingError :analyzed_entity_type entity-type :analyzed_entity_id entity-id))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj b/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.cljindex 1a68b578756..d0129ed652d 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/dependency_types.clj@@ -26,7 +26,7 @@(-> model->dependency-type vals set))(def backfillable-dependency-types- "Dependency types that the backfill job computes dependencies for. Excludes `:table`: tables aren't+ "Dependency types that the dependency job computes dependencies for. Excludes `:table`: tables aren'tbackfilled directly; links involving tables are found via analysis of the other side of the relation."(disj dependency-types :table))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.cljdeleted file mode 100644index 990c09a3d52..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj+++ /dev/null@@ -1,178 +0,0 @@-(ns metabase-enterprise.dependencies.findings- (:require- [metabase-enterprise.dependencies.analysis :as deps.analysis]- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]- [metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.util.log :as log]- [metabase.util.malli :as mu]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(defmulti ^:private instance-db-id- "Gets the database id for a toucan instance"- {:arglists '([toucan-instance])}- t2/model)--(defmethod instance-db-id :default- [toucan-instance]- (:database_id toucan-instance))--(defmethod instance-db-id :model/Transform- [toucan-instance]- (or (some-> toucan-instance :source :query lib/database-id)- ;; non-query transforms (e.g. Python) have no query to read the database from- (:source_database_id toucan-instance)))--(defmethod instance-db-id :model/Segment- [toucan-instance]- (some-> toucan-instance :definition lib/database-id))--(defmulti ^:private pre-analysis-errors- "Return a (possibly empty) sequence of errors detected without running the full analyzer. When- non-empty, the entity is recorded broken with these errors and the regular analysis is skipped.- Use this for structural failures e.g. an orphan Transform whose source database has been deleted."- {:arglists '([toucan-instance])}- t2/model)--(defmethod pre-analysis-errors :default [_] nil)--(defmethod pre-analysis-errors :model/Transform- [{db-id :source_database_id id :id}]- (when (nil? db-id)- [(-> (lib/validation-exception-error "Source database for this transform has been deleted.")- (assoc :source-entity-type :transform :source-entity-id id))]))--(defn upsert-analysis!- "Given a Toucan entity, run its analysis and write the results into `:model/AnalysisFinding`.-- If any row exists already, it is replaced. If it does not exist, it is created."- [toucan-instance]- (when-not (lib-be/metadata-provider-cache)- (throw (ex-info "FIXME: deps.findings/upsert-analysis! ran without reusing `MetadataProvider`s"- {:instance toucan-instance})))- (let [model (t2/model toucan-instance)- entity-type (deps.dependency-types/model->dependency-type model)- instance-id (:id toucan-instance)]- (if-let [errors (seq (pre-analysis-errors toucan-instance))]- (deps.analysis-finding/upsert-analysis! entity-type instance-id false errors)- (if-let [db-id (instance-db-id toucan-instance)]- (let [mp (lib-be/application-database-metadata-provider db-id)- results (try (deps.analysis/check-entity mp entity-type instance-id)- (catch Exception e- (log/errorf "Error analyzing entity: %s" (ex-message e))- [(lib/validation-exception-error (.getMessage e))]))- success (empty? results)]- (deps.analysis-finding/upsert-analysis! entity-type instance-id success results))- ;; No resolvable database, and no pre-analysis error explaining it: record a terminal error so- ;; the entity gets a finding (clearing its stale flag) instead of no-oping forever and being- ;; re-selected on every run. It re-checks normally if its database later becomes resolvable.- (deps.analysis-finding/upsert-analysis!- entity-type instance-id false- [(lib/validation-exception-error "Could not resolve a database for this entity.")])))))--(defn analyze-instances!- "Given a series of toucan entities, upsert analyses for all of them and catch errors."- [instances]- (doseq [instance instances]- (try (upsert-analysis! instance)- (catch Exception e- (log/errorf "Analyzing entity %s %s failed: %s"- (t2/model instance) (:id instance) (ex-message e))))))--(def analyzable-entities- "Entities for which we can compute analysis findings."- #{:card :transform :segment})--(def ^:private AnalyzableEntityType- "Schema for entity types supported by analysis findings."- (into [:enum] analyzable-entities))--(def dependable-entities- "Entities which can be depended on by other entities, even if they cannot themselves be analyzed.-- For example, `:table`s can be passed to [[mark-transitive-dependents-stale!]] as an upstream entity whose dependents are now- stale, but we cannot analyze tables themselves."- (conj analyzable-entities :table))--(defn- mark-supported-dependents-stale!- "Given a map of {entity-type [entity-ids]}, mark all supported (analyzable) types as stale.- For non-analyzable entities (e.g., tables), looks through to their dependents so we still- reach analyzable entities beyond them.- Returns true if any supported dependents were found."- [dependents]- (let [{analyzable true non-analyzable false}- (group-by (fn [[dep-type _]] (boolean (analyzable-entities dep-type)))- (filter (fn [[_ dep-ids]] (seq dep-ids)) dependents))- ;; For non-analyzable entities, find their direct dependents- pass-through-dependents- (when (seq non-analyzable)- (let [key-seq (for [[dep-type dep-ids] non-analyzable- id dep-ids]- [dep-type id])- deps-map (models.dependency/direct-dependents key-seq)]- (models.dependency/group-nodes (into #{} cat (vals deps-map)))))- ;; Merge the analyzable dependents with the pass-through dependents- all-supported (merge-with into- (into {} analyzable)- (into {}- (filter (fn [[dep-type _]] (analyzable-entities dep-type)))- pass-through-dependents))]- (deps.analysis-finding/mark-stale! all-supported)- (boolean (seq all-supported))))--(defn mark-transitive-dependents-stale!- "Mark all transitive dependents of the given upstream entities stale for re-analysis. Takes `{entity-type [entity-ids]}`- — entity types may be non-analyzable (e.g. `:table`), since they can still have analyzable dependents, and each id- collection may be any seqable (vector, set, ...). Returns true if any supported (analyzable) dependents were found.-- Uses `transitive-dependents` (not `transitive-mbql-dependents`) because:- 1. It's faster - doesn't need to fetch entities to check for native-ness- 2. Native dependents marked stale will just be quickly re-validated as valid"- [type->entity-ids]- (-> (models.dependency/transitive-dependents- (update-vals type->entity-ids (fn [ids] (map (fn [id] {:id id}) ids))))- mark-supported-dependents-stale!))--(mu/defn mark-entity-stale!- "Mark a single entity as stale for re-analysis by the background job."- [entity-type :- AnalyzableEntityType- entity-id :- pos-int?]- (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))--(mu/defn mark-entity-and-transitive-dependents-stale!- "Mark `entity-id` and all of its transitive dependents stale for re-analysis. Use this when an entity changes: it- queues the whole affected subtree up front so the entity-check job drains a fixed set of stale entities (see- [[metabase-enterprise.dependencies.task.entity-check/check-entities!]]) without re-propagating staleness during the- drain — which can never terminate when the dependency graph has a cycle."- [entity-type :- AnalyzableEntityType- entity-id :- pos-int?]- (mark-entity-stale! entity-type entity-id)- (mark-transitive-dependents-stale! {entity-type [entity-id]}))--(mu/defn analyze-batch! :- nat-int?- "Add or update analyses for a batch of entities.-- Takes in an entity type and batch size, and looks for a batch of entities with missing or out of date- AnalysisFindings and then upsert new analyses for them.-- This only *analyzes* (each [[upsert-analysis!]] clears that entity's stale flag); it does NOT propagate staleness to- dependents. Propagation happens up front when an entity changes (see [[mark-entity-and-transitive-dependents-stale!]] and the- backfill job), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeing- termination even when the dependency graph has cycles."- [type :- AnalyzableEntityType- batch-size :- pos-int?]- ;; The cache spans the select too: reading an entity attaches a `MetadataProvider` to its query- ;; (see `metabase.lib-be.models.transforms/transform-query`), so selecting outside the cache gives every entity in- ;; the batch a private one.- (lib-be/with-metadata-provider-cache- (let [instances (deps.analysis-finding/instances-for-analysis type batch-size)]- (doseq [instance instances]- (try (upsert-analysis! instance)- (catch Exception e- (log/errorf "Analyzing entity %s %s failed: %s"- (t2/model instance) (:id instance) (ex-message e)))))- (count instances))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.cljindex 9b042052562..d2453f9d283 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj@@ -1,253 +1,326 @@(ns metabase-enterprise.dependencies.hooks- "Implements the hooks of `metabase.dependencies.hooks`, marking written content for the background jobs to recompute."+ "Implements the hooks of `metabase.dependencies.hooks`: a write marks the statuses it makes stale in its own+ transaction, and triggers the background jobs once it commits."(:require[metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.task.backfill :as task.backfill]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]+ [metabase-enterprise.dependencies.task.dependency :as task.dependency]+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata][metabase.app-db.core :as mdb][metabase.batch-processing.core :as grouper]- [metabase.models.interface :as mi][metabase.premium-features.core :refer [defenterprise]][metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defn- trigger-jobs-now!- "Trigger the backfill and entity-check jobs, logging rather than throwing a failure."+(def ^:private job-triggers+ "The functions that trigger the jobs: the dependency job, the analysis finding job, and the result metadata job."+ [#'task.dependency/trigger-dependency-job!+ #'task.analysis-finding/trigger-analysis-finding-job!+ #'task.result-metadata/trigger-result-metadata-job!])++(mu/defn- trigger-jobs-now!+ "Trigger every job, logging rather than throwing a failure."[]- (try- (task.backfill/trigger-backfill-job!)- (task.entity-check/trigger-entity-check-job!)- (catch Throwable e- (log/warnf "Failed to trigger the dependency jobs: %s" (ex-message e)))))--(defn- process-writes!- "Mark the dependents of the Tables changed by `writes` for re-analysis, then trigger the jobs."- [writes]- (when-let [table-ids (not-empty (into #{} (keep :table-id) writes))]- (deps.findings/mark-transitive-dependents-stale! {:table table-ids}))- (trigger-jobs-now!))+ (doseq [trigger! job-triggers]+ (try+ (trigger!)+ (catch Throwable e+ (log/warnf "Failed to trigger %s: %s" (name (symbol trigger!)) (ex-message e))))))(def ^:private writes-queue- (delay (grouper/start! process-writes! :capacity 500 :interval 1000)))+ (delay (grouper/start! (fn [_writes] (trigger-jobs-now!)) :capacity 500 :interval 1000)))-(defn- submit-after-commit!- "Submit `write` to the batch processed once the current transaction commits."- [write]- (mdb/do-after-commit #(grouper/submit! @writes-queue write)))+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}+(def ^:private ^:dynamic *batched-marks*+ "Within [[do-with-batched-marks]], an atom of what the batch marks when it ends: the `:dependents` of+ `{entity-type #{entity-id}}`, the `:remapped-tables` and the `:remap-targets` fields, the only way the hooks of a+ batch can mark each of them once."+ nil)-(defn- trigger-jobs!- "Have the backfill and entity-check jobs triggered for this write, once it commits, together with the others in its- batch, unless a serdes load triggers them once it finishes."+(mu/defn- trigger-jobs!+ "Have every job triggered once the current transaction commits, together with the other writes in its batch."[]- (when-not mi/*deserializing?*- (submit-after-commit! :ok)))--(defn- mark-changed!- "Mark the entity `entity-type` `entity-id` for recalculation and, when analyzable, for re-analysis; the backfill job- marks its dependents for re-analysis."- [entity-type entity-id]- (when entity-id- (deps.dependency-status/mark-stale! {entity-type #{entity-id}})- (when (contains? deps.findings/analyzable-entities entity-type)- (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))- (trigger-jobs!)))--(defn- changed?- "Whether the Toucan update of `instance` changes any of `columns`."- [instance columns]- (let [changes (t2/changes instance)]- (some #(contains? changes %) columns)))--(defn- mark-entity-changed!- "Mark the `model` entity `instance` for recalculation when its update changes any of `columns`."- [model instance columns]- (when (changed? instance columns)- (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))--(defmulti ^:private before-insert*- "Mark what inserting the `model` row `instance` affects."- {:arglists '([model instance])}- (fn [model _instance] model))--(defmulti ^:private before-update*- "Mark what updating the `model` row `instance` affects."- {:arglists '([model instance])}- (fn [model _instance] model))--(defmulti ^:private before-delete*- "Mark what deleting the `model` row `instance` affects."- {:arglists '([model instance])}- (fn [model _instance] model))--(defmethod before-insert* :default [_model _instance] nil)-(defmethod before-update* :default [_model _instance] nil)-(defmethod before-delete* :default [_model _instance] nil)--(doseq [model [:model/Action :model/Card :model/Dashboard :model/Document :model/Measure :model/NativeQuerySnippet- :model/Sandbox :model/Segment :model/Transform]]- (defmethod before-insert* model- [_model _instance]- (trigger-jobs!))- (defmethod before-delete* model- [model instance]- (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))--(doseq [[model columns] {:model/Action #{:model_id :type}- :model/Card #{:type :dataset_query :dimension_mappings :parameters}- :model/Dashboard #{:parameters}- :model/Document #{:content_type :document}- :model/Measure #{:definition :table_id}- :model/NativeQuerySnippet #{:content :template_tags}- :model/Sandbox #{:card_id}- :model/Segment #{:definition :table_id}- :model/Transform #{:source :target}}]- (defmethod before-update* model- [model instance]- (mark-entity-changed! model instance columns)))--(defmethod before-insert* :model/QueryAction- [_model query-action]- (mark-changed! :action (:action_id query-action)))--(defmethod before-update* :model/QueryAction- [_model query-action]- (when (changed? query-action #{:dataset_query})- (mark-changed! :action (:action_id query-action))))--(defmethod before-insert* :model/DashboardCard- [_model dashcard]- (mark-changed! :dashboard (:dashboard_id dashcard)))--(defmethod before-update* :model/DashboardCard- [_model dashcard]- (when (changed? dashcard #{:card_id :dashboard_id :visualization_settings})- (doseq [dashboard-id (distinct [(:dashboard_id (t2/original dashcard)) (:dashboard_id dashcard)])]- (mark-changed! :dashboard dashboard-id))))--(defmethod before-delete* :model/DashboardCard- [_model dashcard]- (mark-changed! :dashboard (:dashboard_id dashcard)))--(defmethod before-insert* :model/DashboardCardSeries- [_model series]- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))--(defmethod before-update* :model/DashboardCardSeries- [_model series]- (when (changed? series #{:card_id :dashboardcard_id})- (doseq [dashcard-id (distinct [(:dashboardcard_id (t2/original series)) (:dashboardcard_id series)])]- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id dashcard-id)))))--(defmethod before-delete* :model/DashboardCardSeries- [_model series]- (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))--(defmethod before-delete* :model/DashboardTab- [_model tab]- (mark-changed! :dashboard (:dashboard_id tab)))--(defn- mark-table-changed!- "Mark the dependents of the Table with `table-id` for re-analysis once the current transaction commits."- [table-id]- (when table-id- (submit-after-commit! {:table-id table-id})))--(defmethod before-insert* :model/TableUserSettings- [_model settings]- (mark-table-changed! (:table_id settings)))--(defmethod before-update* :model/TableUserSettings- [_model settings]- (mark-table-changed! (:table_id settings)))--(defmethod before-delete* :model/TableUserSettings- [_model settings]- (mark-table-changed! (:table_id settings)))--(defmethod before-insert* :model/FieldUserSettings- [_model settings]- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))--(defmethod before-update* :model/FieldUserSettings- [_model settings]- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))--(defmethod before-delete* :model/FieldUserSettings- [_model settings]- (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))--(defmethod before-insert* :model/Dimension- [_model dimension]- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))--(defmethod before-update* :model/Dimension- [_model dimension]- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))--(defmethod before-delete* :model/Dimension- [_model dimension]- (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))--(defmethod before-delete* :model/Database- [_model database]- (doseq [transform-id (dependencies.db/transform-ids-of-source-database (:id database))]- (mark-changed! :transform transform-id)))+ (when-not *batched-marks*+ (mdb/do-after-commit #(grouper/submit! @writes-queue {:dependency-jobs true}))))++(def ^:private Row+ "A row a hook is given, a Toucan instance."+ [:fn {:error/message "a row"} map?])++(def ^:private Columns+ "The columns a write of a model has to change, or `:any`."+ [:maybe [:or [:= :any] [:set :keyword]]])++(def ^:private Node+ "An entity as `[entity-type entity-id]`, the ID nil when unknown."+ [:maybe [:tuple ::deps.dependency-types/dependency-types [:maybe ms/PositiveInt]]])++(mu/defn- changed? :- :boolean+ "Whether the Toucan update of `row` changes any of `columns`, or any column at all when `columns` is `:any`."+ [row :- Row+ columns :- Columns]+ (let [changes (t2/changes row)]+ (if (= :any columns)+ (boolean (seq changes))+ (boolean (some #(contains? changes %) columns)))))++(def ^:private dependency-columns+ "For each model, the columns whose update has the dependency job compute the entity's dependencies again."+ {:model/Action #{:model_id :type}+ :model/Card #{:type :dataset_query :dimension_mappings :parameters}+ :model/Dashboard #{:parameters}+ :model/Document #{:content_type :document}+ :model/Measure #{:definition :table_id}+ :model/NativeQuerySnippet #{:content :template_tags}+ :model/Sandbox #{:card_id}+ :model/Segment #{:definition :table_id}+ :model/Transform #{:source :target}})++(def ^:private analysis-columns+ "For each model, the columns whose update has the analysis job analyze the entity itself again."+ {:model/Card #{:type :dataset_query}+ :model/Segment #{:definition :table_id}+ :model/Transform #{:source :target}})++(def ^:private model-output-columns+ "For each model, the columns whose update changes what the entities directly depending on it read, so their statuses+ are marked."+ {:model/Card #{:type :result_metadata}+ :model/Measure #{:definition :table_id}+ :model/NativeQuerySnippet #{:content :template_tags}+ :model/Segment #{:definition :table_id}+ :model/Table #{:name :schema :display_name :active :visibility_type}})++(mu/defn- output-columns :- Columns+ "The [[model-output-columns]] of `row` of `model`, with the name of a metric, which the cards aggregating it show."+ [model :- :keyword+ row :- Row]+ (cond-> (get model-output-columns model)+ (and (= :model/Card model) (= :metric (keyword (:type row)))) (conj :name)))++(def ^:private field-columns+ "The Field columns whose update changes what the entities using the Field's Table read."+ #{:name :display_name :description :active :visibility_type :settings :base_type :effective_type :database_type+ :semantic_type :coercion_strategy :fk_target_field_id :position})++(mu/defn- part-owner :- Node+ "The entity, as `[entity-type entity-id]`, that `row` of `model` is a part of, or nil."+ [model :- :keyword+ row :- [:maybe Row]]+ (case model+ (:model/DashboardCard :model/DashboardTab) [:dashboard (:dashboard_id row)]+ :model/DashboardCardSeries [:dashboard (some-> (:dashboardcard_id row)+ dependencies.db/dashcard-dashboard-id)]+ :model/QueryAction [:action (:action_id row)]+ (:model/Field :model/TableUserSettings) [:table (:table_id row)]+ :model/FieldUserSettings [:table (some-> (:field_id row) dependencies.db/field-table-id)]+ nil))++(def ^:private part-columns+ "For each model whose rows are parts of an entity, the columns whose update changes the entity."+ {:model/DashboardCard #{:card_id :dashboard_id :visualization_settings}+ :model/DashboardCardSeries #{:card_id :dashboardcard_id}+ :model/DashboardTab #{}+ :model/QueryAction #{:dataset_query}+ :model/Field field-columns+ :model/TableUserSettings #{:display_name :visibility_type :data_layer :field_order}+ :model/FieldUserSettings (conj field-columns :custom_position :nfc_path :json_unfolding)})++(mu/defn- mark-dependents!+ "Mark the result metadata and analysis statuses of the entities directly depending on the entity `entity-type`+ `entity-id` stale."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity-id :- ms/PositiveInt]+ (if-let [batched *batched-marks*]+ (swap! batched update-in [:dependents entity-type] (fnil conj #{}) entity-id)+ (do+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! entity-type #{entity-id})+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! entity-type #{entity-id}))))++(mu/defn- mark-remapped-tables!+ "Mark the result metadata and analysis statuses of every entity transitively depending on the Tables with `table-ids`+ stale, whose remapped columns their stored metadata carries down."+ [table-ids :- [:set ms/PositiveInt]]+ (when (seq table-ids)+ (if-let [batched *batched-marks*]+ (swap! batched update :remapped-tables (fnil into #{}) table-ids)+ (let [dependents (models.dependency/transitive-dependents {:table (mapv #(hash-map :id %) table-ids)})]+ (dependencies.db/mark-result-metadata-statuses-stale! dependents)+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents)))))++(mu/defn- mark-remap-targets!+ "Mark what reads the remaps whose human-readable field is one of the Fields with `field-ids`, a change of which the+ remapped columns show."+ [field-ids :- [:set ms/PositiveInt]]+ (when (seq field-ids)+ (if-let [batched *batched-marks*]+ (swap! batched update :remap-targets (fnil into #{}) field-ids)+ (mark-remapped-tables! (dependencies.db/remapped-table-ids field-ids)))))++(mu/defn- mark-dimension!+ "Mark what reads the remap of the Dimension `row`, whose Field's Table's entities show it."+ [row :- [:maybe Row]]+ (when-let [table-id (some-> (:field_id row) dependencies.db/field-table-id)]+ (mark-remapped-tables! #{table-id})))++(mu/defn- mark-field-values! :- :boolean+ "Mark what reads the internal remap of the Field of the FieldValues, which shows only while they have human-readable+ values, when a write of them as `before` and `after` adds or removes those."+ [before :- [:maybe Row]+ after :- [:maybe Row]]+ (let [row (or after before)+ readable (comp boolean seq :human_readable_values)]+ (boolean+ (when (and (= :full (keyword (:type row)))+ (not= (readable before) (readable after))+ (dependencies.db/internal-remap? (:field_id row)))+ (mark-dimension! row)+ true))))++(mu/defn- mark-output-changed!+ "Mark what reads the output of the entity `entity-type` `entity` stale: the direct dependents, or every transitive+ dependent of a measure or a snippet, which their dependents expand and nothing between them stores."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity :- Row]+ (case entity-type+ :measure (let [dependents (models.dependency/transitive-dependents {:measure [entity]})]+ (dependencies.db/mark-result-metadata-statuses-stale! dependents)+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents))+ :snippet (let [dependents (models.dependency/transitive-dependents {:snippet [entity]})]+ (dependencies.db/mark-dependency-statuses-stale! dependents)+ (dependencies.db/mark-analysis-finding-statuses-stale! dependents))+ (mark-dependents! entity-type (:id entity))))++(mu/defn- mark-part-owner!+ "Mark what a write of a part of the `[owner-type owner-id]` entity changes: the dependencies of a dashboard or an+ action, or what the entities depending on a table read."+ [[owner-type owner-id] :- Node]+ (when owner-id+ (if (= :table owner-type)+ (mark-dependents! :table owner-id)+ (dependencies.db/mark-dependency-statuses-stale! {owner-type #{owner-id}}))))++(mu/defn- output-changed? :- :boolean+ "Whether the update of `row` of `model` changes what the entities directly depending on it read, ignoring a card's+ result metadata changes they don't see, such as a run's new fingerprints."+ [model :- :keyword+ row :- Row]+ (let [columns (output-columns model row)]+ (and (changed? row columns)+ (or (not= :model/Card model)+ (changed? row (disj columns :result_metadata))+ (not (deps.result-metadata/same-output? (:result_metadata (t2/original row)) (:result_metadata row)))))))++(mu/defn- mark-update! :- :boolean+ "Mark the statuses an update of `row` of `model` makes stale: the entity's own, by the columns each job+ reads, and what reads its output when that changes, returning whether a column some job reads changed."+ [model :- :keyword+ row :- Row]+ (let [entity-type (deps.dependency-types/model->dependency-type model)+ own {entity-type #{(:id row)}}+ dependency? (changed? row (dependency-columns model))+ analysis? (changed? row (analysis-columns model))+ output? (output-changed? model row)+ part? (changed? row (part-columns model))]+ (when dependency?+ (dependencies.db/mark-dependency-statuses-stale! own))+ (when analysis?+ (dependencies.db/mark-analysis-finding-statuses-stale! own))+ (when output?+ (mark-output-changed! entity-type row))+ (when part?+ (run! mark-part-owner! (distinct [(part-owner model (t2/original row)) (part-owner model row)])))+ (when part?+ (case model+ :model/Field (mark-remap-targets! #{(:id row)})+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})+ nil))+ (or dependency? analysis? output? part?+ (case model+ :model/Dimension (do (run! mark-dimension! (distinct [(t2/original row) row])) true)+ :model/FieldValues (mark-field-values! (t2/original row) row)+ false))))++(defenterprise do-with-batched-marks+ "Run `thunk`, marking what its writes change once per entity after it ends, even when it throws, instead of once per+ row in each write's transaction, and triggering the jobs once."+ :feature :dependencies+ [thunk]+ (if *batched-marks*+ (thunk)+ (let [batched (atom {})]+ (try+ (binding [*batched-marks* batched]+ (thunk))+ (finally+ (let [{:keys [dependents remapped-tables remap-targets]} @batched]+ (doseq [[entity-type entity-ids] dependents]+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! entity-type entity-ids)+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! entity-type entity-ids))+ (mark-remapped-tables! (into (set remapped-tables)+ (when (seq remap-targets)+ (dependencies.db/remapped-table-ids remap-targets)))))+ (trigger-jobs!))))))(defenterprise before-insert- "Mark what inserting the `model` row `instance` affects for recalculation."+ "Mark what inserting `row` of `model` changes, and trigger the jobs once it commits; a new entity has no+ status, which has the jobs process it.":feature :dependencies- [model instance]- (before-insert* model instance))+ [model row]+ (when (contains? part-columns model)+ (mark-part-owner! (part-owner model row)))+ (case model+ :model/Dimension (mark-dimension! row)+ :model/FieldValues (mark-field-values! nil row)+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})+ nil)+ (trigger-jobs!))(defenterprise before-update- "Mark what updating the `model` row `instance` affects for recalculation."+ "Mark what updating `row` of `model` changes, and trigger the jobs once it commits when it marked any.":feature :dependencies- [model instance]- (before-update* model instance))+ [model row]+ (when (mark-update! model row)+ (trigger-jobs!)))(defenterprise before-delete- "Mark what deleting the `model` row `instance` affects for recalculation."- :feature :dependencies- [model instance]- (before-delete* model instance))--(defenterprise after-database-sync- "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed, and the- entities reading from it that name tables that matched nothing, which sync may have found now."+ "Mark what deleting `row` of `model` changes: the entity it is a part of, what reads a deleted entity, or+ the transforms reading a deleted Database; and trigger the jobs once it commits.":feature :dependencies- [database-id]- (doseq [[entity-type entity-ids] (dependencies.db/unresolved-table-entity-ids database-id)- entity-id entity-ids]- (mark-changed! entity-type entity-id))- (when (deps.findings/mark-transitive-dependents-stale!- {:table (dependencies.db/table-ids-with-outdated-findings database-id)})- (trigger-jobs!)))+ [model row]+ (when (contains? part-columns model)+ (mark-part-owner! (part-owner model row)))+ (case model+ :model/Dimension (mark-dimension! row)+ :model/FieldValues (mark-field-values! row nil)+ :model/Field (mark-remap-targets! #{(:id row)})+ :model/FieldUserSettings (mark-remap-targets! #{(:field_id row)})+ nil)+ (when-let [entity-type (deps.dependency-types/model->dependency-type model)]+ (mark-output-changed! entity-type row))+ (when (= model :model/Database)+ (when-let [transform-ids (not-empty (dependencies.db/source-database-transform-ids (:id row)))]+ (dependencies.db/mark-dependency-statuses-stale! {:transform transform-ids})+ (dependencies.db/mark-analysis-finding-statuses-stale! {:transform transform-ids})))+ (trigger-jobs!))(defenterprise after-transform-run- "Make the output Table with `table-id` depend on the transform with `transform-id`, and mark the transform's- dependents for re-analysis."+ "Make the output Table with `table-id` depend on the transform with `transform-id`.":feature :dependencies[transform-id table-id](when table-id(models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))- (when (deps.findings/mark-transitive-dependents-stale! {:transform #{transform-id}})- (trigger-jobs!)))--(defenterprise after-serdes-load- "Trigger the backfill and entity-check jobs for what a serdes load marked."- :feature :dependencies- []- (trigger-jobs-now!))+ (trigger-jobs!))(defenterprise after-premium-token-update- "Restart the backfill and entity-check jobs, which stop while the dependencies feature is disabled."+ "Restart every job, since they stop while the dependencies feature is disabled.":feature :dependencies[](trigger-jobs-now!))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.cljindex 29087660a17..f6637ad3a92 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj@@ -3,5 +3,6 @@[metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.schema][metabase-enterprise.dependencies.settings]- [metabase-enterprise.dependencies.task.backfill]- [metabase-enterprise.dependencies.task.entity-check]))+ [metabase-enterprise.dependencies.task.analysis-finding]+ [metabase-enterprise.dependencies.task.dependency]+ [metabase-enterprise.dependencies.task.result-metadata]))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.cljindex 0a3dfbbd278..30df6d1906e 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj@@ -2,8 +2,11 @@(:require[metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]+ [metabase.lib.schema.validate :as lib.schema.validate][metabase.models.interface :as mi][metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical][toucan2.core :as t2]))@@ -12,7 +15,7 @@(derive :model/AnalysisFinding :metabase/model)(t2/deftransforms :model/AnalysisFinding- {:analyzed_entity_type mi/transform-keyword})+ {:entity_type mi/transform-keyword})(t2/define-before-insert :model/AnalysisFinding[finding]@@ -42,61 +45,30 @@- 8: Flag MBQL queries whose source-table is missing/inactive (e.g. dropped during sync)"8)-(defn- error->finding-error-row+(mu/defn- error->finding-error-row :- ::deps.analysis-finding-error/error-input"Convert an error from lib/find-bad-refs-with-source to a row for analysis_finding_error table.We use `:message` for the error detail if there was an `validation-exception-error`."- [error]+ [error :- ::lib.schema.validate/error-with-source]{:error-type (:type error);; error-detail cannot be longer than 254:error-detail (some-> (or (:name error) (:message error)) (u/truncate 254)):source-entity-type (:source-entity-type error):source-entity-id (:source-entity-id error)})-(defn upsert-analysis!+(def ^:private FindingDetails+ "The errors an analysis found, as `lib/find-bad-refs-with-source` returns them."+ [:maybe [:set ::lib.schema.validate/error-with-source]])++(mu/defn upsert-analysis!"Given the details of an AnalysisFinding row, upsert the data into the actual db.Also writes individual errors to the analysis_finding_error table with source information."- [type instance-id result finding-details]+ [entity-type :- :keyword+ entity-id :- ms/PositiveInt+ result :- :boolean+ finding-details :- FindingDetails](t2/with-transaction [_conn]- (let [update {:analysis_version *current-analysis-finding-version*- :result result- :stale false}- existing-id (dependencies.db/finding-id type instance-id)]- (if existing-id- (dependencies.db/update-finding! existing-id update)- (dependencies.db/insert-finding! (assoc update- :analyzed_entity_type type- :analyzed_entity_id instance-id)))- (deps.analysis-finding-error/replace-errors-for-entity!- type- instance-id- (map error->finding-error-row finding-details)))))--(def ^:private mark-stale-batch-size- "Maximum number of entity IDs to process in a single query to avoid parameter limits."- 1000)--(defn mark-stale!- "Mark the existing findings of the entities in `type->ids`, `{entity-type ids}`, stale for re-analysis. Entities- without findings are ignored - they'll be picked up by the normal job flow."- [type->ids]- (doseq [[entity-type ids] type->ids- batch (partition-all mark-stale-batch-size ids)]- (dependencies.db/mark-findings-stale! entity-type batch)))--(defn has-stale-entities?- "Check if there are any stale analysis records."- []- (dependencies.db/stale-finding-exists?))--(defn stale-entity-count- "Number of analysis findings currently marked stale, across all entity types. Used by the entity-check drain loop to- detect whether it is still making progress."- []- (dependencies.db/stale-finding-count))--(defn instances-for-analysis- "Find a batch of instances of type `entity-type` and maximum size `batch-size` with missing, outdated,- or stale AnalysisFindings.- Prioritizes stale entities (stale=true) over outdated version entities."- [entity-type batch-size]- (dependencies.db/instances-for-analysis entity-type batch-size *current-analysis-finding-version*))+ (dependencies.db/upsert-finding! entity-type entity-id result)+ (deps.analysis-finding-error/replace-errors-for-entity!+ entity-type+ entity-id+ (map error->finding-error-row finding-details))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.cljnew file mode 100644index 00000000000..c4b0b88b2ca--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_status.clj@@ -0,0 +1,13 @@+(ns metabase-enterprise.dependencies.models.analysis-finding-status+ "The AnalysisFindingStatus model: one status per analyzable entity, marked stale by the writes the analysis reads."+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/AnalysisFindingStatus [_model] :analysis_finding_status)++(derive :model/AnalysisFindingStatus :metabase/model)++(t2/deftransforms :model/AnalysisFindingStatus+ {:entity_type mi/transform-keyword})diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 5812611cb8f..da302e456f7 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -3,6 +3,7 @@[clojure.set :as set][metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.schema :as dependencies.schema][metabase-enterprise.sandbox.schema][metabase.actions.schema :as actions.schema][metabase.documents.schema :as documents.schema]@@ -17,6 +18,8 @@[metabase.transforms.schema :as transforms.schema][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical][potemkin :as p][toucan2.core :as t2]))@@ -44,7 +47,35 @@{:from_entity_type mi/transform-keyword:to_entity_type mi/transform-keyword})-(defn- deps-children+(mr/def ::entity-with-id+ "Any map with an optional `:id`, whatever else it holds."+ [:map {::mr/deliberately-open true, :description "a Toucan, metadata, or user-supplied entity"}+ [:id {:optional true} [:maybe ::deps.dependency-types/entity-id]]])++(def ^:private EntitiesMap+ "Entities as `{entity-type [entity-or-id]}`, nil for none of a type: each one an entity ID, or a map whose `:id` is+ one (skipped when nil), such as a Toucan entity, a `MetadataProvider` entity, or user input."+ [:map-of+ ::deps.dependency-types/dependency-types+ [:maybe+ [:or+ [:sequential [:or ::deps.dependency-types/entity-id ::entity-with-id]]+ [:set [:or ::deps.dependency-types/entity-id ::entity-with-id]]]]])++(def ^:private UncheckedEntityId+ "A dependency entity ID as a caller gives it, which [[ensure-entity-id]] checks."+ [:schema {::mr/deliberately-open true, :description "an unchecked dependency entity id"} :any])++(def ^:private DependenciesByType+ "The dependencies of an entity as `{entity-type #{entity-id}}`, with the IDs not yet checked."+ [:map-of ::dependencies.schema/entity-type [:set UncheckedEntityId]])++(def ^:private KeyFn+ "[[key-dependents]] or [[key-dependencies]], called with nodes and the two edge restrictions."+ [:-> [:sequential ::dependencies.schema/node] ::dependencies.schema/edge-restriction ::dependencies.schema/edge-restriction+ ::dependencies.schema/child-map])++(mu/defn- deps-children :- ::dependencies.schema/child-map"Get dependency children with optional visibility filtering.Returns a map from [src-type src-id] tuples to sets of [dst-type dst-id] tuples representing dependencies.@@ -53,20 +84,28 @@`edge-restriction-expr`: nil, `{:visible {...}}`, `{:broken? true}`, `{:types #{...}}`, or`{:entity-type ... :ids #{...}}`) restricting the destination (child) and source (parent) side of each edge,respectively."- ([{:keys [src-type src-id dst-type dst-id key-seq destination-filter source-filter]}]- (transduce (map (fn [[entity-type entity-keys]]- (let [deps (dependencies.db/dependency-edges- {:src-type src-type :src-id src-id :dst-type dst-type :dst-id dst-id- :entity-type entity-type :entity-ids entity-keys- :destination-restriction destination-filter :source-restriction source-filter})]- (u/group-by (juxt src-type src-id)- (juxt dst-type dst-id)- conj #{}- deps))))- merge {}- (u/group-by first second key-seq))))--(defn- key-dependents+ [{:keys [src-type src-id dst-type dst-id key-seq destination-filter source-filter]}+ :- [:map {:closed true}+ [:src-type [:enum :from_entity_type :to_entity_type]]+ [:src-id [:enum :from_entity_id :to_entity_id]]+ [:dst-type [:enum :from_entity_type :to_entity_type]]+ [:dst-id [:enum :from_entity_id :to_entity_id]]+ [:key-seq [:sequential ::dependencies.schema/node]]+ [:destination-filter ::dependencies.schema/edge-restriction]+ [:source-filter ::dependencies.schema/edge-restriction]]]+ (transduce (map (fn [[entity-type entity-keys]]+ (let [deps (dependencies.db/dependency-edges+ {:src-type src-type :src-id src-id :dst-type dst-type :dst-id dst-id+ :entity-type entity-type :entity-ids entity-keys+ :destination-restriction destination-filter :source-restriction source-filter})]+ (u/group-by (juxt src-type src-id)+ (juxt dst-type dst-id)+ conj #{}+ deps))))+ merge {}+ (u/group-by first second key-seq)))++(mu/defn- key-dependents :- ::dependencies.schema/child-map"Get the dependent entity keys for the entity keys in `key-seq`.Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys@@ -74,9 +113,11 @@`destination-filter` and `source-filter` are filter specs restricting the dependent (child) and queried (parent)side of each edge, respectively — see [[deps-children]]."- ([key-seq]+ ([key-seq :- [:sequential ::dependencies.schema/node]](key-dependents key-seq nil nil))- ([key-seq destination-filter source-filter]+ ([key-seq :- [:sequential ::dependencies.schema/node]+ destination-filter :- ::dependencies.schema/edge-restriction+ source-filter :- ::dependencies.schema/edge-restriction](deps-children{:src-type :to_entity_type:src-id :to_entity_id@@ -86,12 +127,12 @@:destination-filter destination-filter:source-filter source-filter})))-(defn direct-dependents+(mu/defn direct-dependents :- ::dependencies.schema/child-map"Returns the direct dependents for the given entity key sequence."- [key-seq]+ [key-seq :- [:sequential ::dependencies.schema/node]](key-dependents key-seq))-(defn- key-dependencies+(mu/defn- key-dependencies :- ::dependencies.schema/child-map"Get the dependency entity keys for the entity keys in `key-seq`.Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys@@ -99,9 +140,11 @@`destination-filter` and `source-filter` are filter specs restricting the dependency (child) and queried (parent)side of each edge, respectively — see [[deps-children]]."- ([key-seq]+ ([key-seq :- [:sequential ::dependencies.schema/node]](key-dependencies key-seq nil nil))- ([key-seq destination-filter source-filter]+ ([key-seq :- [:sequential ::dependencies.schema/node]+ destination-filter :- ::dependencies.schema/edge-restriction+ source-filter :- ::dependencies.schema/edge-restriction](deps-children{:src-type :from_entity_type:src-id :from_entity_id@@ -114,56 +157,60 @@(p/deftype+ DependencyGraph [children-fn]graph/Graph(children-of [_this key-seq]- (children-fn key-seq)))+ (children-fn (vec key-seq))))-(defn graph-dependents+(mu/defn graph-dependents :- ::graph/graph"Return a dependency graph for finding dependents (downstream entities)."[](->DependencyGraph key-dependents))-(defn graph-dependencies+(mu/defn graph-dependencies :- ::graph/graph"Return a dependency graph for finding dependencies (upstream entities)."[](->DependencyGraph key-dependencies))-(defn- filtered-graph+(mu/defn- filtered-graph :- ::graph/graph"Create a dependency graph with visibility filtering.Arguments:- `key-fn`: Either key-dependencies or key-dependents, determining graph direction- `destination-filter`: Filter spec restricting the child side of a dependency — see [[deps-children]]- `source-filter`: Filter spec restricting the parent side of a dependency"- [key-fn destination-filter source-filter]+ [key-fn :- KeyFn+ destination-filter :- ::dependencies.schema/edge-restriction+ source-filter :- ::dependencies.schema/edge-restriction](->DependencyGraph(fn [key-seq](key-fn key-seq destination-filter source-filter))))-(defn filtered-graph-dependencies+(mu/defn filtered-graph-dependencies :- ::graph/graph"Create a permission-aware dependency graph for finding upstream dependencies.Arguments:- `destination-filter`: Filter spec restricting destination entities — see [[deps-children]]- `source-filter`: Optional filter spec restricting the parent side of a dependency"- ([destination-filter]+ ([destination-filter :- ::dependencies.schema/edge-restriction](filtered-graph-dependencies destination-filter nil))- ([destination-filter source-filter]+ ([destination-filter :- ::dependencies.schema/edge-restriction+ source-filter :- ::dependencies.schema/edge-restriction](filtered-graph key-dependencies destination-filter source-filter)))-(defn filtered-graph-dependents+(mu/defn filtered-graph-dependents :- ::graph/graph"Create a permission-aware dependency graph for finding downstream dependents.Arguments:- `destination-filter`: Filter spec restricting destination entities — see [[deps-children]]- `source-filter`: Optional filter spec restricting the parent side of a dependency"- ([destination-filter]+ ([destination-filter :- ::dependencies.schema/edge-restriction](filtered-graph-dependents destination-filter nil))- ([destination-filter source-filter]+ ([destination-filter :- ::dependencies.schema/edge-restriction+ source-filter :- ::dependencies.schema/edge-restriction](filtered-graph key-dependents destination-filter source-filter)))-(defn entities->nodes+(mu/defn entities->nodes :- [:sequential ::dependencies.schema/node]"Converts a map of entities `{entity-type [{:id 1, ...} ...]}` or entity IDs `{entity-type [1]}` into a list of nodes`[[entity-type entity-id]]`."- [entities-map]+ [entities-map :- EntitiesMap](for [[entity-type entities] entities-mapentity entities:let [id (if (number? entity)@@ -172,12 +219,12 @@:when id][entity-type id]))-(defn group-nodes+(mu/defn group-nodes :- ::dependencies.schema/type->ids"Groups a list of nodes `[[entity-type entity-id]]` by their type."- [nodes]+ [nodes :- [:sequential ::dependencies.schema/node]](u/group-by first second conj #{} nodes))-(defn transitive-dependents+(mu/defn transitive-dependents :- ::dependencies.schema/type->ids"Given a map of entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependentsas `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.@@ -187,8 +234,10 @@`MetadataProvider` entities, user input, etc.**Excludes** the input entities from the list of dependents!"- ([entities-map] (transitive-dependents nil entities-map))- ([graph entities-map]+ ([entities-map :- EntitiesMap]+ (transitive-dependents nil entities-map))+ ([graph :- [:maybe ::graph/graph]+ entities-map :- EntitiesMap](let [graph (or graph (graph-dependents))starters (entities->nodes entities-map)](->> (graph/transitive graph starters) ; This returns a flat list.@@ -227,27 +276,33 @@:action (= :query (:type entity))false)))-(defn- native-lookup-map [children]+(mu/defn- native-lookup-map :- [:map-of ::dependencies.schema/node :boolean]+ "Whether each node in `children` is native."+ [children :- ::dependencies.schema/child-map](let [grouped (-> (graph/all-map-nodes children)group-nodes)](into {}(mapcat (fn [[node-type ids]]- (mapv (fn [entity]- [[node-type (:id entity)]- (is-native-entity? node-type entity)])- (dependencies.db/instances node-type ids))))+ (if (= node-type :card)+ (let [native (dependencies.db/native-card-ids ids)]+ (map (fn [id] [[:card id] (contains? native id)]) ids))+ (mapv (fn [entity]+ [[node-type (:id entity)]+ (is-native-entity? node-type entity)])+ (dependencies.db/entities node-type ids)))))grouped)))-(defn transitive-mbql-dependents+(mu/defn transitive-mbql-dependents :- ::dependencies.schema/type->ids"Equivalent to `transitive-dependents`, except it excludes any native cards/transforms/segments and their children.Also, the order is more flexible (though consistent between runs).Note that this does not check the passed in entities for native-ness -- the filter is only applied to theirtransitive children."- ([entities-map]+ ([entities-map :- EntitiesMap](transitive-mbql-dependents nil entities-map))- ([graph entities-map]+ ([graph :- [:maybe ::graph/graph]+ entities-map :- EntitiesMap](let [start-nodes (set (entities->nodes entities-map))children (graph/transitive-children-of (or graph (graph-dependents)) (seq start-nodes))native-lookup (native-lookup-map children)]@@ -259,17 +314,35 @@:else node))children)))))-(defn- ensure-entity-id- [id]+(mu/defn- ensure-entity-id :- ::deps.dependency-types/entity-id+ "`id`, throwing a 400 when it is not a positive integer."+ [id :- UncheckedEntityId](when-not (pos-int? id)(throw (ex-info "Dependency entity id must be a positive integer"{:status-code 400, :id id})))id)-(defn replace-dependencies!- "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with- the ones specified in `dependencies-by-type`. "- [entity-type entity-id dependencies-by-type]+(mu/defn new-dependencies :- [:sequential [:tuple ::dependencies.schema/entity-type ::deps.dependency-types/entity-id]]+ "The dependencies in `dependencies-by-type` that the entity of type `entity-type` with id `entity-id` doesn't have+ yet, as `[[entity-type entity-id]]`."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ dependencies-by-type :- DependenciesByType]+ (let [current (into #{}+ (map (juxt :to_entity_type :to_entity_id))+ (dependencies.db/dependencies-from entity-type entity-id))]+ (vec (for [[to-entity-type ids] dependencies-by-type+ to-entity-id ids+ :let [node [to-entity-type (ensure-entity-id to-entity-id)]]+ :when (not (current node))]+ node))))++(mu/defn replace-dependencies! :- [:maybe :int]+ "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with the ones specified in+ `dependencies-by-type`, returning the number inserted, or nil when none are."+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ dependencies-by-type :- DependenciesByType](let [current-dependencies (dependencies.db/dependencies-from entity-type entity-id)to-remove (keep (fn [{:keys [id to_entity_type to_entity_id]}](when-not (get-in dependencies-by-type [to_entity_type to_entity_id])@@ -289,7 +362,7 @@(when (seq to-add)(dependencies.db/insert-dependencies! to-add)))))-(defn swap-dependency!+(mu/defn swap-dependency! :- :int"Efficiently swap a dependency from old-source to new-source during replacement operations.This is more efficient than full dependency analysis since we know exactly what changed.@@ -301,7 +374,10 @@- entity-id: The ID of the entity- old-source: The source being replaced, as [source-type source-id] (e.g., [:card 783])- new-source: The new source, as [source-type source-id] (e.g., [:table 164])"- [entity-type entity-id [old-source-type old-source-id] [new-source-type new-source-id]]+ [entity-type :- ::dependencies.schema/entity-type+ entity-id :- ms/PositiveInt+ [old-source-type old-source-id] :- [:tuple ::dependencies.schema/entity-type ms/PositiveInt]+ [new-source-type new-source-id] :- [:tuple ::dependencies.schema/entity-type ms/PositiveInt]](let [already-present? (dependencies.db/dependency-exists? entity-type entity-id new-source-type new-source-id)](if already-present?(dependencies.db/delete-dependency! entity-type entity-id old-source-type old-source-id)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.cljindex 362646d79eb..47c19f712d1 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj@@ -1,10 +1,12 @@(ns metabase-enterprise.dependencies.models.dependency-status(:require- [java-time.api :as t][metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.schema :as dependencies.schema][metabase.models.interface :as mi]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical][toucan2.core :as t2]))@@ -15,88 +17,49 @@(t2/deftransforms :model/DependencyStatus{:entity_type mi/transform-keyword})-(defn mark-stale!- "Mark the entities in `type->ids`, `{entity-type ids}`, stale for dependency recalculation, creating their statuses- if they don't exist and resetting their retry state so previously-failed entities get a fresh chance."- [type->ids]- (doseq [[entity-type ids] type->ids- id ids]- (dependencies.db/mark-dependency-status-stale! entity-type id)))--(defn upsert-status!- "Upsert a dependency_status entry, setting stale=false, version to current, whether the entity names tables that- match nothing, and clearing any failure state."- [entity-type entity-id unresolved-tables?]- (dependencies.db/upsert-dependency-status!- entity-type entity-id models.dependency/current-dependency-analysis-version unresolved-tables?))-(defmulti hydrate-for-deps- "Hydrate a batch of instances with data needed for dependency calculation.+ "Hydrate a batch of entities with data needed for dependency calculation.Dispatches on entity-type keyword. Default is identity (no hydration needed)."- {:arglists '([entity-type instances])}- (fn [entity-type _instances] entity-type))+ {:arglists '([entity-type entities])}+ (fn [entity-type _entities] entity-type))-(defmethod hydrate-for-deps :default [_ instances] instances)+(defmethod hydrate-for-deps :default [_ entities] entities)-(defmethod hydrate-for-deps :dashboard [_ instances]- (t2/hydrate instances [:dashcards :series]))+(defmethod hydrate-for-deps :dashboard [_ entities]+ (t2/hydrate entities [:dashcards :series]))-(defmethod hydrate-for-deps :action [_ instances]- (let [queries (when (seq instances)- (dependencies.db/query-action-queries (mapv :id instances)))]+(defmethod hydrate-for-deps :action [_ entities]+ (let [queries (when (seq entities)+ (dependencies.db/query-action-queries (mapv :id entities)))](mapv (fn [{:keys [id] :as action}](cond-> action(contains? queries id) (assoc :dataset_query (queries id))))- instances)))+ entities)))-(defn instances-for-dependency-calculation- "Find a batch of instances of type `entity-type` and maximum size `batch-size` that need- dependency calculation: no status row yet, stale=true, OR version < current.- Excludes terminal entities and entities whose retry delay hasn't elapsed.- Returns full entity objects. Prioritizes stale over outdated.- Uses Java time (not DB time) so tests with [[mt/with-clock]] work correctly."- [entity-type batch-size]- (dependencies.db/instances-for-dependency-calculation entity-type- batch-size- models.dependency/current-dependency-analysis-version- (t/offset-date-time)))+(mu/defn take-pending-statuses! :- [:sequential ::dependencies.schema/pending-status]+ "Up to `batch-size` entities of `entity-type` whose dependencies a write changed, are not computed yet, or were+ computed by an older analysis, those last, with the `stale` count the job clears afterwards."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (dependencies.db/take-pending-dependency-statuses!+ entity-type batch-size models.dependency/current-dependency-analysis-version))-(defn has-pending-retries?- "Returns true if there are any entities waiting to be retried (not terminal, with a set retry time)."- []- (dependencies.db/pending-retry-exists?))+(mu/defn clear-status! :- :boolean+ "Record that the dependency job computed the dependencies of the entity `entity-type` `entity-id`, clearing its+ `stale` count unless a write raised it."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity-id :- ms/PositiveInt+ stale :- ms/IntGreaterThanOrEqualToZero]+ (dependencies.db/clear-dependency-status!+ entity-type entity-id stale models.dependency/current-dependency-analysis-version))-(defn has-stale-or-outdated?- "Returns true if there are any entities needing dependency calculation: no status row yet,- stale=true, OR version < current (not terminal, retry delay elapsed), or deleted entities whose status the backfill- has yet to clean up. Defined in terms of [[instances-for-dependency-calculation]] so it stays consistent with what- the backfill processes."+(mu/defn has-pending-entities? :- :boolean+ "Returns true if there are any entities whose dependencies are stale, not computed yet or computed by an older+ analysis, or deleted entities whose status the dependency job has yet to clean up."[](boolean(some (fn [entity-type]- (or (seq (instances-for-dependency-calculation entity-type 1))- (seq (dependencies.db/deleted-entity-ids entity-type 1))))+ (or (dependencies.db/dependency-pending?+ entity-type models.dependency/current-dependency-analysis-version)+ (seq (dependencies.db/dependency-deleted-entity-ids entity-type 1))))deps.dependency-types/backfillable-dependency-types)))--(defn record-failure!- "Record a failed dependency calculation attempt for an entity.- Increments fail_count and sets next_retry_at based on exponential backoff.- If max retries exceeded, marks the entity as terminal.- Creates the entry if it doesn't exist, since entities with no row yet are exactly the ones- [[instances-for-dependency-calculation]] picks up first."- [entity-type entity-id max-retries delay-minutes]- (dependencies.db/record-dependency-status-failure!- entity-type entity-id- (fn [existing]- ;; An inserted row takes the column defaults for `stale` (false) and `dependency_analysis_version` (0). 0 is- ;; below `current-dependency-analysis-version`, which is what keeps the entity eligible for the retry once- ;; `next_retry_at` has elapsed.- (let [new-fail-count (inc (:fail_count existing 0))]- (if (> new-fail-count max-retries)- {:fail_count new-fail-count- :terminal true- :next_retry_at nil}- (let [retry-minutes (* new-fail-count delay-minutes)]- {:fail_count new-fail-count- :next_retry_at (when (pos? retry-minutes)- (t/plus (t/offset-date-time) (t/minutes retry-minutes)))}))))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.cljnew file mode 100644index 00000000000..2c3e5bdd8e5--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/result_metadata_status.clj@@ -0,0 +1,21 @@+(ns metabase-enterprise.dependencies.models.result-metadata-status+ "The ResultMetadataStatus model: one status per MBQL card, marked stale by the writes its result metadata reads."+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/ResultMetadataStatus [_model] :result_metadata_status)++(derive :model/ResultMetadataStatus :metabase/model)++(t2/deftransforms :model/ResultMetadataStatus+ {:entity_type mi/transform-keyword})++(def current-result-metadata-version+ "Current version of the result metadata logic. Increment it when the logic changes, so the job processes every+ entity again.++ Version history:+ - 1: Initial version"+ 1)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.clj b/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.cljnew file mode 100644index 00000000000..d6a2c2690ad--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/result_metadata.clj@@ -0,0 +1,160 @@+(ns metabase-enterprise.dependencies.result-metadata+ "Re-derives the result metadata of the MBQL cards whose ResultMetadataStatus a write marked stale, which a write of+ what they read does; native cards are left alone, since their result metadata comes from running them."+ (:require+ [clojure.walk :as walk]+ [malli.error :as me]+ [metabase-enterprise.dependencies.db :as dependencies.db]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.result-metadata-status :as models.result-metadata-status]+ [metabase-enterprise.dependencies.schema :as dependencies.schema]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.queries.core :as queries]+ [metabase.types.core :as types]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private compared-column-keys+ "The result metadata column keys that the queries using a card as a source see."+ [:name :display_name :description :semantic_type :fk_target_field_id :id :settings :visibility_type+ :base_type :effective_type :unit :inherited_temporal_unit])++(def ^:private measured-column-keys+ "The result metadata column keys a run of the card measures, within the type that inference gets right."+ [:base_type :effective_type :database_type])++(def ^:private Column+ "A result metadata column, as a card stores or inference returns it, valid or not."+ [:fn {:error/message "a map"} map?])++(def ^:private TypeName+ "A type, as a keyword or as the string a stored column may hold it as."+ [:maybe [:or :keyword :string]])++(mu/defn- normalize-column :- Column+ "The result metadata column `col` with its unqualified keys in snake_case."+ [col :- Column]+ (update-keys col #(cond-> % (not (qualified-keyword? %)) u/->snake_case_en)))++(mu/defn- comparable-metadata :- [:vector [:map-of :keyword :any]]+ "The [[compared-column-keys]] of the result metadata `cols` with a value, with keyword values as strings."+ [cols :- [:sequential Column]]+ (mapv (fn [col]+ (into {}+ (keep (fn [k]+ (when-some [v (get col k)]+ [k (walk/postwalk #(cond-> % (keyword? %) u/qualified-name) v)])))+ compared-column-keys))+ cols))++(mu/defn same-output? :- :boolean+ "Whether the result metadata `before` and `after` show the queries using the card the same columns."+ [before :- [:maybe [:sequential Column]]+ after :- [:maybe [:sequential Column]]]+ (= (comparable-metadata (map normalize-column before))+ (comparable-metadata (map normalize-column after))))++(mu/defn- same-root-type? :- [:maybe :boolean]+ "Whether the types `x` and `y` share an ancestor below `:type/*`, like integers and decimals do."+ [x :- TypeName+ y :- TypeName]+ (and x y (not= :type/* (types/most-specific-common-ancestor (keyword x) (keyword y)))))++(mu/defn- measured-by-a-run? :- [:maybe :boolean]+ "Whether the stored column `stored-col` holds what a run measured for the inferred column `col`: a run stored its+ source type, they have types of the same root, and the column's source type is unchanged or unknown."+ [stored-col :- Column+ col :- Column]+ (and (some? (:database_type stored-col))+ (same-root-type? (:base_type stored-col) (:base_type col))+ (or (nil? (:database_type col))+ (= (:database_type col) (:database_type stored-col)))))++(mu/defn- patch-result-metadata :- [:vector Column]+ "The inferred result metadata `inferred`, each column keeping the [[measured-column-keys]] of the stored column of+ `stored` it stands for when a run measured them, and only that column's fingerprint, so that the job never shows a+ fingerprint the card's runs and saves did not store."+ [stored :- [:sequential Column]+ inferred :- [:sequential Column]]+ (mapv (fn [col stored-col]+ (cond-> (dissoc col :fingerprint)+ (and stored-col (measured-by-a-run? stored-col col))+ (merge (select-keys stored-col measured-column-keys))++ (some? (:fingerprint stored-col))+ (assoc :fingerprint (:fingerprint stored-col))))+ inferred+ (lib/match-returned-columns stored inferred)))++(mu/defn- rederived-result-metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]+ "The re-derived result metadata of the MBQL `card`, inferred as no user so no one's permissions decide it, or nil+ when what the queries using it see is unchanged."+ [{query :dataset_query, stored :result_metadata, :keys [id], :as card} :- (ms/InstanceOf :model/Card)]+ (let [stored (mapv normalize-column stored)+ metadata (patch-result-metadata+ stored+ (or (queries/infer-metadata-with-model-overrides query card)+ (throw (ex-info "Could not infer the result metadata of the card" {:card-id id}))))]+ (when (not= (comparable-metadata stored) (comparable-metadata metadata))+ (when-let [error (me/humanize (mr/explain ::lib.schema.metadata/card.result-metadata metadata))]+ (throw (ex-info "Invalid re-derived result metadata" {:card-id id :error error})))+ metadata)))++(mu/defn- store-result-metadata!+ "Mark the entities directly depending on the Card with `card-id` stale, then, under the card's row lock, store its+ re-derived result metadata `metadata` through the Card model when the card is still in the `state` it was re-derived+ from, or else mark the card stale, so it is re-derived from what a save or a run stored meanwhile."+ [card-id :- ::lib.schema.id/card+ state :- ::dependencies.schema/card-state+ metadata :- ::lib.schema.metadata/card.result-metadata]+ (t2/with-transaction [_]+ (dependencies.db/mark-dependents-result-metadata-statuses-stale! :card #{card-id})+ (dependencies.db/mark-dependents-analysis-finding-statuses-stale! :card #{card-id})+ (if (= state (dependencies.db/card-state card-id true))+ (dependencies.db/set-card-result-metadata! card-id metadata)+ (dependencies.db/mark-result-metadata-statuses-stale! {:card #{card-id}}))))++(mu/defn- rederive-card!+ "Re-derive the result metadata of the MBQL Card with `card-id` and store it when it changed."+ [card-id :- ::lib.schema.id/card]+ (let [state (dependencies.db/card-state card-id false)+ card (dependencies.db/card card-id)+ metadata (lib-be/with-metadata-provider-cache+ (when (and card (not= :native (:query_type card)) (seq (:dataset_query card)))+ (rederived-result-metadata card)))]+ (when (and state metadata)+ (store-result-metadata! card-id state metadata))))++(mu/defn update-result-metadata! :- nat-int?+ "Delete the statuses of deleted cards, then re-derive up to `batch-size` MBQL cards whose ResultMetadataStatus counts+ a write, is missing or is outdated, clearing each one's `stale` count unless a write raised it meanwhile, or counting+ its failure, returning how many statuses it took, deleted ones included."+ [batch-size :- ms/PositiveInt]+ (let [deleted-ids (dependencies.db/result-metadata-deleted-card-ids batch-size)+ statuses (dependencies.db/take-pending-result-metadata-statuses!+ batch-size models.result-metadata-status/current-result-metadata-version+ models.dependency/current-dependency-analysis-version)]+ (when (seq deleted-ids)+ (dependencies.db/delete-result-metadata-statuses! deleted-ids))+ (doseq [{card-id :entity_id, :keys [stale]} statuses]+ (try+ (rederive-card! card-id)+ (dependencies.db/clear-result-metadata-status!+ card-id stale models.result-metadata-status/current-result-metadata-version)+ (catch Throwable e+ (log/warnf "Failed to re-derive the result metadata of card %s: %s" card-id (ex-message e))+ (when (dependencies.db/record-result-metadata-failure! card-id (deps.settings/dependency-max-failures))+ (log/warnf "Giving up on the result metadata of card %s until it changes" card-id))+ (when-not (instance? Exception e)+ (throw e)))))+ (+ (count deleted-ids) (count statuses))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.cljindex fe54439a1d9..71a29b23408 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj@@ -2,9 +2,61 @@(:require[metabase.documents.schema :as documents.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.queries.schema :as queries.schema][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::entity-type+ "A dependency-type keyword, or the equivalent string, which the keyword transform of every entity type column coerces."+ [:or :metabase-enterprise.dependencies.dependency-types/dependency-types :string])++(mr/def ::node+ "A dependency graph node: an entity as `[entity-type entity-id]`."+ [:tuple+ :metabase-enterprise.dependencies.dependency-types/dependency-types+ :metabase-enterprise.dependencies.dependency-types/entity-id])++(mr/def ::child-map+ "The children of dependency graph nodes, as `{node #{child-node}}`."+ [:map-of ::node [:set ::node]])++(mr/def ::visible-opts+ "The user whose readable entities a query is restricted to (`:user-id`, `:is-superuser?`, `:is-data-analyst?`), and+ whether archived entities count (`:include-archived-items`, default `:exclude`)."+ [:map {:closed true}+ [:user-id ::lib.schema.id/user]+ [:is-superuser? {:optional true} [:maybe :boolean]]+ [:is-data-analyst? {:optional true} [:maybe :boolean]]+ [:include-archived-items {:optional true} [:enum :exclude :all :only]]])++(mr/def ::edge-restriction+ "The restriction of one side of a dependency edge, nil for none: the entities readable by a user (`:visible`), the+ entities that failed analysis (`:broken?`), the entities of some types (`:types`), and the entities of one type with+ some ids (`:entity-type` and `:ids`)."+ [:maybe+ [:map {:closed true}+ [:visible {:optional true} ::visible-opts]+ [:broken? {:optional true} :boolean]+ [:types {:optional true} [:set ::entity-type]]+ [:entity-type {:optional true} ::entity-type]+ [:ids {:optional true} [:sequential :metabase-enterprise.dependencies.dependency-types/entity-id]]]])++(mr/def ::pending-status+ "An entity a job is to process, with the `stale` count of its status when the job picked it, nil without a status."+ [:map {:closed true}+ [:entity_id ms/PositiveInt]+ [:stale [:maybe ms/IntGreaterThanOrEqualToZero]]])++(mr/def ::type->ids+ "Entities as `{entity-type ids}`."+ [:map-of :metabase-enterprise.dependencies.dependency-types/dependency-types [:set ms/PositiveInt]])++(mr/def ::card-state+ "The `updated_at` and `result_metadata` of a Card, which a save or a run changes."+ [:map {:closed true}+ [:updated_at ms/TemporalInstant]+ [:result_metadata [:maybe ::queries.schema/card.result-metadata]]])+(mr/def ::upstream-deps[:map[:card {:optional true} [:set ::lib.schema.id/card]]@@ -14,27 +66,7 @@[:dashboard {:optional true} [:set ::lib.schema.id/dashboard]][:document {:optional true} [:set ::documents.schema/document.id]][:sandbox {:optional true} [:set ::lib.schema.id/sandbox]]- [:segment {:optional true} [:set ::lib.schema.id/segment]]- [:unresolved-table {:optional true} [:set [:map- [:schema {:optional true} :string]- [:table :string]]]]])--(mr/def ::analysis-finding- "A AnalysisFinding as selected from the app DB: every column of `:analysis_finding`."- [:merge- ::analysis-finding.update- [:map {:closed true}- [:id ms/PositiveInt]]])--(mr/def ::analysis-finding.update- "What an update (or insert) of a AnalysisFinding accepts: every column of `:analysis_finding` except `id`, all optional."- [:map {:closed true}- [:analyzed_entity_type {:optional true} [:maybe [:or :keyword :string]]]- [:analyzed_entity_id {:optional true} [:maybe ms/PositiveInt]]- [:analysis_version {:optional true} [:maybe :int]]- [:analyzed_at {:optional true} [:maybe ms/TemporalInstant]]- [:result {:optional true} [:maybe :boolean]]- [:stale {:optional true} [:maybe :boolean]]])+ [:segment {:optional true} [:set ::lib.schema.id/segment]]])(mr/def ::analysis-finding-error"A AnalysisFindingError as selected from the app DB: every column of `:analysis_finding_error`."@@ -67,22 +99,3 @@[:from_entity_id {:optional true} [:maybe ms/PositiveInt]][:to_entity_type {:optional true} [:maybe [:or :keyword :string]]][:to_entity_id {:optional true} [:maybe ms/PositiveInt]]])--(mr/def ::dependency-status- "A DependencyStatus as selected from the app DB: every column of `:dependency_status`."- [:merge- ::dependency-status.update- [:map {:closed true}- [:id ms/PositiveInt]]])--(mr/def ::dependency-status.update- "What an update (or insert) of a DependencyStatus accepts: every column of `:dependency_status` except `id`, all optional."- [:map {:closed true}- [:entity_type {:optional true} [:maybe [:or :keyword :string]]]- [:entity_id {:optional true} [:maybe ms/PositiveInt]]- [:dependency_analysis_version {:optional true} [:maybe :int]]- [:stale {:optional true} [:maybe :boolean]]- [:fail_count {:optional true} [:maybe :int]]- [:next_retry_at {:optional true} [:maybe ms/TemporalInstant]]- [:terminal {:optional true} [:maybe :boolean]]- [:unresolved_tables {:optional true} [:maybe :boolean]]])diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/settings.cljindex c66e6b0eaa4..6bd36a7722d 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj@@ -3,7 +3,7 @@[metabase.settings.core :refer [defsetting]]))(defsetting dependency-backfill-batch-size- "The number of entities that the dependency backfill job will process per run."+ "The number of entities that the dependency job will process per run.":visibility :internal:type :integer:default 500@@ -11,7 +11,7 @@:export? false)(defsetting dependency-backfill-delay-minutes- "The average delay between when the dependency backfill job will run."+ "The average delay before the dependency job runs again after a run that failed.":visibility :internal:type :integer:default 60@@ -19,9 +19,8 @@:export? false)(defsetting dependency-backfill-variance-minutes- "The variation in when the dependency backfill job will run. The actual delay between runs will be a random value- between `dependency-backfill-delay-minutes - dependency-backfill-variance-minutes` and- `dependency-backfill-delay-minutes + dependency-backfill-variance-minutes`."+ "The variation of the dependency job's delay after a failed run, which is a random value within this many minutes of+ `dependency-backfill-delay-minutes`, and the longest random delay of its first run after startup.":visibility :internal:type :integer:default 30@@ -29,7 +28,7 @@:export? false)(defsetting dependency-entity-check-batch-size- "The number of entities that the entity check job will process per run."+ "The number of entities that the analysis finding job will process per run.":visibility :internal:type :integer:default 500@@ -37,7 +36,7 @@:export? false)(defsetting dependency-entity-check-delay-minutes- "The average delay between when the entity check job will run."+ "The average delay before the analysis finding job runs again after a run that failed.":visibility :internal:type :integer:default 60@@ -45,11 +44,44 @@:export? false)(defsetting dependency-entity-check-variance-minutes- "The variation in when the entity check job will run. The actual delay between runs will be a random value- between `dependency-entity-check-delay-minutes - dependency-entity-check-variance-minutes` and- `dependency-entity-check-delay-minutes + dependency-entity-check-variance-minutes`."+ "The variation of the analysis finding job's delay after a failed run, which is a random value within this many+ minutes of `dependency-entity-check-delay-minutes`, and the longest random delay of its first run after startup.":visibility :internal:type :integer:default 30:setter :none:export? false)++(defsetting dependency-result-metadata-batch-size+ "The number of entities that the result metadata job will process per run."+ :visibility :internal+ :type :integer+ :default 500+ :setter :none+ :export? false)++(defsetting dependency-result-metadata-delay-minutes+ "The average delay before the result metadata job runs again after a run that failed."+ :visibility :internal+ :type :integer+ :default 60+ :setter :none+ :export? false)++(defsetting dependency-result-metadata-variance-minutes+ "The variation of the result metadata job's delay after a failed run, which is a random value within this many+ minutes of `dependency-result-metadata-delay-minutes`, and the longest random delay of its first run after startup."+ :visibility :internal+ :type :integer+ :default 30+ :setter :none+ :export? false)++(defsetting dependency-max-failures+ "How many times in a row a dependency job fails to process an entity before it gives up on it until a write changes+ it."+ :visibility :internal+ :type :integer+ :default 3+ :setter :none+ :export? false)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.cljnew file mode 100644index 00000000000..3022b83d074--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/analysis_finding.clj@@ -0,0 +1,53 @@+(ns metabase-enterprise.dependencies.task.analysis-finding+ "The job that runs the analysis of the entities whose AnalysisFindingStatus is stale, missing or outdated."+ (:require+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase-enterprise.dependencies.task.util :as task.util]+ [metabase.premium-features.core :as premium-features]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))++(set! *warn-on-reflection* true)++(mu/defn- run-batch! :- [:maybe ms/PositiveInt]+ "Analyze a batch of entities whose findings are outdated, returning when to run again: a second from now when it took+ any, or nil."+ []+ (when (premium-features/has-feature? :dependencies)+ (let [batch-size (deps.settings/dependency-entity-check-batch-size)+ taken (deps.analysis-finding/analyze-batch! batch-size)]+ (when (pos? taken)+ (log/debug "Processed" taken "entities for analysis")+ 1))))++(declare ^:private job)++(task/defjob UpdateAnalysisFindings+ "Re-analyze the entities that changed or whose upstreams changed."+ {:saved-class "metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"+ :concurrent? false}+ [ctx]+ (log/debug "Executing UpdateAnalysisFindings job...")+ (task.util/run-and-reschedule! (.getScheduler ctx) job))++(def ^:private job+ {:job-type UpdateAnalysisFindings+ :job-key "metabase.dependencies.task.entity-check.job"+ :trigger-key "metabase.dependencies.task.entity-check.trigger"+ :job-name "Update Analysis Findings"+ :batch-size deps.settings/dependency-entity-check-batch-size+ :delay-minutes deps.settings/dependency-entity-check-delay-minutes+ :variance-minutes deps.settings/dependency-entity-check-variance-minutes+ :run-batch! #'run-batch!})++(defmethod task/init! ::UpdateAnalysisFindings [_]+ (task.util/init-job! job))++(mu/defn trigger-analysis-finding-job! :- :any+ "Trigger the UpdateAnalysisFindings job, unless a MB_DEPENDENCY_ENTITY_CHECK_BATCH_SIZE that isn't positive+ disables it."+ []+ (task.util/trigger-run! job))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljdeleted file mode 100644index cda348a3988..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ /dev/null@@ -1,268 +0,0 @@-(ns metabase-enterprise.dependencies.task.backfill- "Implements a task that brings all entities with dependencies tracked in the dependency table- (see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure- the dependency table contains fresh entries.-- This is done by querying the dependency_status table for entities that are stale or have an- outdated dependency_analysis_version. The backfill task computes dependencies and updates- the dependency_status table. Entities that a write marked stale, or that were deleted, also have their dependents- marked for re-analysis here, outside the write's transaction."- (:require- [clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.scheduler :as qs]- [clojurewerkz.quartzite.triggers :as triggers]- [java-time.api :as t]- [metabase-enterprise.dependencies.calculation :as deps.calculation]- [metabase-enterprise.dependencies.db :as dependencies.db]- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.settings :as deps.settings]- [metabase-enterprise.dependencies.task-util :as deps.task-util]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]- [metabase.graph.core :as graph]- [metabase.lib-be.core :as lib-be]- [metabase.premium-features.core :as premium-features]- [metabase.task.core :as task]- [metabase.transforms.core :as transforms]- [metabase.util.log :as log]- [toucan2.core :as t2])- (:import (org.quartz JobExecutionContext)))--(set! *warn-on-reflection* true)--(def ^:private expanded-into-dependents- "The entity types whose content a native query using them expands into its own SQL, so that a change to one changes- the table dependencies of the entities using it, which no graph edge carries."- #{:snippet})--(def ^:private entity-types- "The list of entity types to backfill, those expanded into their dependents first, so that the dependents they mark- are recomputed in the same run.-- This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, links- involving tables are found via analysis of the other side of the relation."- (vec (sort-by (complement expanded-into-dependents) deps.dependency-types/backfillable-dependency-types)))--(def ^:private max-retries 5)--;;; ------------------------------ Post-deps cleanup multimethod --------------------------------(defmulti post-deps-cleanup!- "Perform entity-specific cleanup after dependencies have been replaced.- For example, transforms need to clean up outdated downstream table->transform dependencies.- Default is no-op."- {:arglists '([entity-type entity])}- (fn [entity-type _entity] entity-type))--(defmethod post-deps-cleanup! :default [_ _entity] nil)--(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]- (let [db-id (transforms/transform-source-database transform)- downstream-table-ids (dependencies.db/downstream-table-ids-of-transform id)- downstream-tables (when (seq downstream-table-ids)- (dependencies.db/tables downstream-table-ids))- outdated-tables (remove (fn [table]- (and (= (:schema table) (:schema target))- (= (:name table) (:name target))- (or (not db-id)- (= db-id (:db_id table)))))- downstream-tables)- not-found-table-ids (remove (into #{} (map :id) downstream-tables)- downstream-table-ids)]- (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)- (map :id) outdated-tables))]- (dependencies.db/delete-table-dependencies-on-transform! outdated-downstream-table-ids id))))--;;; ------------------------------ Backfill orchestration --------------------------------(defn- processable-instances [entity-type batch-size]- (deps.dependency-status/hydrate-for-deps- entity-type- (deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))--(defn- mark-expanding-entities-stale!- "Mark for recalculation the entities whose native SQL expands the `entity-type` entity `entity-id`, directly or- through the entities of the same kind that include it."- [entity-type entity-id]- (let [graph (models.dependency/filtered-graph-dependents nil {:types expanded-into-dependents})- type->ids (models.dependency/group-nodes (graph/transitive graph [[entity-type entity-id]]))]- (deps.dependency-status/mark-stale! (apply dissoc type->ids expanded-into-dependents))))--(defn- compute-deps-for-entity!- "Mark the dependents of an entity that a write marked stale for re-analysis, and those that expand it for- recalculation, then compute and store its dependencies and update its dependency_status, flagging the tables it- names that match nothing. Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."- [entity-type entity]- (log/debug "Computing dependencies for" (name entity-type) (:id entity))- (when (:stale (dependencies.db/dependency-status entity-type (:id entity)))- (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}})- (when (expanded-into-dependents entity-type)- (mark-expanding-entities-stale! entity-type (:id entity))))- (t2/with-transaction [_]- (let [deps (deps.calculation/calculate-deps entity-type entity)]- (models.dependency/replace-dependencies! entity-type (:id entity) (dissoc deps :unresolved-table))- (post-deps-cleanup! entity-type entity)- (deps.dependency-status/upsert-status! entity-type (:id entity) (contains? deps :unresolved-table)))))--(defn- remove-deleted-entity-batch!- "Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop their- dependencies and status, returning how many were removed."- [entity-type batch-size]- (let [entity-ids (dependencies.db/deleted-entity-ids entity-type batch-size)]- (when (seq entity-ids)- (t2/with-transaction [_]- (deps.findings/mark-transitive-dependents-stale! {entity-type entity-ids})- (dependencies.db/delete-dependencies-from! entity-type entity-ids)- (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))- (count entity-ids)))--(defn- backfill-entity-batch!- [entity-type batch-size]- (lib-be/with-metadata-provider-cache- (let [type-name (name entity-type)- instances (processable-instances entity-type batch-size)]- (when (seq instances)- (log/infof "Processing a batch of %s %s(s)..." (count instances) type-name))- (reduce (fn [total entity]- (+ total- (try- (compute-deps-for-entity! entity-type entity)- 1- (catch Throwable e ;; catch OOMs- (let [id (:id entity)]- (try- (deps.dependency-status/record-failure!- entity-type id max-retries- (deps.settings/dependency-backfill-delay-minutes))- (let [{:keys [fail_count terminal]} (dependencies.db/dependency-status entity-type id)]- (if terminal- (log/errorf "Entity %s %s failed %d times, marking as terminally broken: %s"- type-name id fail_count (ex-message e))- (log/warnf "Entity %s %s failed, failure count: %d. %s"- type-name id fail_count (ex-message e))))- (catch Exception record-ex- (log/errorf "Entity %s %s failed during dependency calculation: %s"- type-name id (ex-message e))- (log/errorf "Additionally, failed to record the failure for %s %s: %s"- type-name id (ex-message record-ex)))))- (when-not (instance? Exception e) ;; re-throw Errors- (throw e))- 0))))- 0- instances))))--(defn- backfill-dependencies!- "Job to backfill dependencies for all entities, triggering the entity-check job for the findings it marked stale.- Returns true if a full batch has been selected, nil or false otherwise."- []- (when (premium-features/has-feature? :dependencies)- (let [batch-size (deps.settings/dependency-backfill-batch-size)- remaining (reduce (fn [remaining entity-type]- (if (< remaining 1)- (reduced 0)- (let [removed (remove-deleted-entity-batch! entity-type remaining)- processed (if (< removed remaining)- (backfill-entity-batch! entity-type (- remaining removed))- 0)]- (when (pos? processed)- (log/info "Updated" processed "entities."))- (- remaining removed processed))))- batch-size- entity-types)]- (when (< remaining batch-size)- (task.entity-check/trigger-entity-check-job!))- (< remaining 1))))--(defn- has-pending-retries?- "Whether any entity is in retry backoff that this instance can actually act on.-- Fails open on an indeterminate token status. `has-feature?` cannot tell a network failure from being unlicensed, and- both inputs to the job's reschedule decision consult the licence, so reading a blip as `false` would stop the job- for good — nothing re-fires when the check recovers, because the token never changed. Only a definitive `false`- counts as unlicensed here — which is the common case, since no token at all answers `false`. A token that cannot be- validated answers `nil` and keeps the job alive. Doing the work still fails closed — see- [[backfill-dependencies!]]."- []- (and (not (false? (premium-features/canonically-has-feature? :dependencies)))- (deps.dependency-status/has-pending-retries?)))--(declare schedule-run!)--(defn- next-run-delay []- (deps.task-util/job-delay- (deps.settings/dependency-backfill-delay-minutes)- (deps.settings/dependency-backfill-variance-minutes)))--(defn- run-and-reschedule!- "Process a batch, then schedule the next run if there is more to do.-- Reschedules on the way out of a failure too, then rethrows. This job is one-shot self-rescheduling with no cron- backstop, so anything escaping the batch would otherwise end the chain until a content-change event or a restart —- survivable for an `OutOfMemoryError` that takes the process with it, not for a `StackOverflowError` from- pathological SQL that leaves a live process behind."- [scheduler]- (try- (let [full-batch-selected? (backfill-dependencies!)- retries? (has-pending-retries?)]- (if (or full-batch-selected? retries?)- (schedule-run! scheduler (next-run-delay))- (log/info "No more entities to backfill for, stopping.")))- (catch Throwable e- (schedule-run! scheduler (next-run-delay))- (throw e))))--(defn- log-job-start- [^JobExecutionContext ctx]- (let [scheduler (.getScheduler ctx)- job-key (.getKey (.getJobDetail ctx))- job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))- active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]- (log/infof "Executing %s job. %s total triggers active." job-class active-triggers-count)))--(task/defjob BackfillDependencies- "Backfill the dependency table."- {:saved-class "metabase_enterprise.dependencies.task.backfill.BackfillDependencies"- :concurrent? false}- [ctx]- (let [ctx ^JobExecutionContext ctx]- (log-job-start ctx)- (run-and-reschedule! (.getScheduler ctx))))--(def ^:private job-key "metabase.task.dependency-backfill.job")-(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")--(defn- schedule-run!- "Schedule a run of the backfill job `delay-in-seconds` from now."- [scheduler delay-in-seconds]- (let [start-at (-> (t/instant)- (t/+ (t/duration delay-in-seconds :seconds))- java.util.Date/from)- trigger (triggers/build- (triggers/with-identity (triggers/key trigger-key))- (triggers/for-job job-key)- (triggers/start-at start-at))- job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]- (log/info "Scheduling next run of job Dependency Backfill at" start-at)- (task/schedule-task! scheduler job trigger)))--(defn trigger-backfill-job!- "Trigger the BackfillDependencies job to run after a brief delay, unless the batch size disables it.-- The 1-second delay ensures the calling transaction has committed before the job checks for stale entities. Entity- changes fire this, so leaving it ungated is what made a disabled job wake roughly once a second; the job's own- periodic schedule is left alone so it still resumes if the batch size becomes positive.-- Disable with MB_DEPENDENCY_BACKFILL_BATCH_SIZE=0"- []- (when (pos? (deps.settings/dependency-backfill-batch-size))- (schedule-run! (task/scheduler) 1)))--(defmethod task/init! ::DependencyBackfill [_]- (when-not (pos? (deps.settings/dependency-backfill-batch-size))- (log/info "Dependency backfill batch size is not positive; the job will run but process nothing"))- (schedule-run!- (task/scheduler)- (deps.task-util/job-initial-delay- (deps.settings/dependency-backfill-variance-minutes))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.cljnew file mode 100644index 00000000000..f5219d4843d--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/dependency.clj@@ -0,0 +1,200 @@+(ns metabase-enterprise.dependencies.task.dependency+ "The job that computes the dependencies of the entities whose dependency_status counts a write, records an older+ analysis_version or is missing, clearing each count afterwards unless a write raised it meanwhile."+ (:require+ [clojurewerkz.quartzite.scheduler :as qs]+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]+ [metabase-enterprise.dependencies.calculation :as deps.calculation]+ [metabase-enterprise.dependencies.db :as dependencies.db]+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]+ [metabase-enterprise.dependencies.task.util :as task.util]+ [metabase.lib-be.core :as lib-be]+ [metabase.premium-features.core :as premium-features]+ [metabase.task.core :as task]+ [metabase.transforms.core :as transforms]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2])+ (:import (org.quartz JobExecutionContext)))++(set! *warn-on-reflection* true)++(def ^:private entity-types+ "The list of entity types whose dependencies the job computes, snippets first, so that the snippets the entities+ using them expand are known when those are computed.++ This is not the same as deps.dependency-types/dependency-types, because tables have no dependencies of their+ own. Instead, links involving tables are found via analysis of the other side of the relation."+ (vec (sort-by #(not= :snippet %) deps.dependency-types/backfillable-dependency-types)))++(def ^:private Entity+ "An entity whose dependencies the job computes, as the app DB holds it."+ [:fn {:error/message "a Toucan entity"} t2/instance?])++;;; ------------------------------ Post-deps cleanup multimethod ------------------------------++(defmulti post-deps-cleanup!+ "Perform entity-specific cleanup after dependencies have been replaced.+ For example, transforms need to clean up outdated downstream table->transform dependencies.+ Default is no-op."+ {:arglists '([entity-type entity])}+ (fn [entity-type _entity] entity-type))++(defmethod post-deps-cleanup! :default [_ _entity] nil)++(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]+ (let [db-id (transforms/transform-source-database transform)+ downstream-table-ids (dependencies.db/downstream-table-ids-of-transform id)+ downstream-tables (when (seq downstream-table-ids)+ (dependencies.db/tables downstream-table-ids))+ outdated-tables (remove (fn [table]+ (and (= (:schema table) (:schema target))+ (= (:name table) (:name target))+ (or (not db-id)+ (= db-id (:db_id table)))))+ downstream-tables)+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)+ downstream-table-ids)]+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)+ (map :id) outdated-tables))]+ (dependencies.db/delete-table-dependencies-on-transform! outdated-downstream-table-ids id))))++;;; ------------------------------ Job orchestration ------------------------------++(mu/defn- hydrated-entities :- [:sequential Entity]+ "The entities of `entity-type` with `entity-ids`, hydrated for computing their dependencies."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity-ids :- [:set ms/PositiveInt]]+ (vec (deps.dependency-status/hydrate-for-deps+ entity-type (vec (dependencies.db/entities entity-type entity-ids)))))++(mu/defn- compute-deps-for-entity! :- :any+ "Compute and store the dependencies of an `entity` hydrated by [[deps.dependency-status/hydrate-for-deps]], marking+ an entity with a new upstream stale for its result metadata and analysis, or for a snippet the analysis of what+ expands it, which an upstream change made before the edge existed never marked."+ [entity-type :- ::deps.dependency-types/dependency-types+ entity :- Entity]+ (log/debug "Computing dependencies for" (name entity-type) (:id entity))+ (let [deps (deps.calculation/calculate-deps entity-type entity)+ own {entity-type #{(:id entity)}}]+ (t2/with-transaction [_]+ (when (seq (models.dependency/new-dependencies entity-type (:id entity) deps))+ (when (= :card entity-type)+ (dependencies.db/mark-result-metadata-statuses-stale! own))+ (when (contains? deps.analysis-finding/analyzable-entities entity-type)+ (dependencies.db/mark-analysis-finding-statuses-stale! own))+ (when (= :snippet entity-type)+ (dependencies.db/mark-analysis-finding-statuses-stale!+ (models.dependency/transitive-dependents {:snippet [entity]}))))+ (models.dependency/replace-dependencies! entity-type (:id entity) deps)+ (post-deps-cleanup! entity-type entity))))++(mu/defn- remove-deleted-entity-batch! :- ms/IntGreaterThanOrEqualToZero+ "Drop the dependencies and status of up to `batch-size` deleted entities of `entity-type`, returning how many were+ removed."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (let [entity-ids (dependencies.db/dependency-deleted-entity-ids entity-type batch-size)]+ (when (seq entity-ids)+ (t2/with-transaction [_]+ (dependencies.db/delete-dependencies-from! entity-type entity-ids)+ (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))+ (count entity-ids)))++(mu/defn- update-entity-batch! :- ms/IntGreaterThanOrEqualToZero+ "Compute the dependencies of up to `batch-size` pending entities of `entity-type`, clearing each one's `stale` count+ unless a write raised it meanwhile, or counting its failure, returning how many it took."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (lib-be/with-metadata-provider-cache+ (let [type-name (name entity-type)+ statuses (deps.dependency-status/take-pending-statuses! entity-type batch-size)+ stale (into {} (map (juxt :entity_id :stale)) statuses)]+ (when (seq statuses)+ (log/debugf "Processing a batch of %s %s(s)..." (count statuses) type-name))+ (doseq [entity (when (seq statuses) (hydrated-entities entity-type (set (keys stale))))]+ (try+ (compute-deps-for-entity! entity-type entity)+ (deps.dependency-status/clear-status! entity-type (:id entity) (stale (:id entity)))+ (catch Throwable e ;; catch OOMs+ (log/errorf "Computing the dependencies of %s %s failed: %s" type-name (:id entity) (ex-message e))+ (when (dependencies.db/record-dependency-failure! entity-type (:id entity)+ (deps.settings/dependency-max-failures))+ (log/warnf "Giving up on the dependencies of %s %s until it changes" type-name (:id entity)))+ (when-not (instance? Exception e) ;; re-throw Errors+ (throw e)))))+ (count statuses))))++(mu/defn- update-dependencies! :- [:maybe :boolean]+ "Update the dependencies of up to a batch of entities, triggering the jobs that read them, and return whether the+ batch took any."+ []+ (when (premium-features/has-feature? :dependencies)+ (let [batch-size (deps.settings/dependency-backfill-batch-size)+ remaining (reduce (fn [remaining entity-type]+ (if (< remaining 1)+ (reduced 0)+ (let [removed (remove-deleted-entity-batch! entity-type remaining)+ processed (if (< removed remaining)+ (update-entity-batch! entity-type (- remaining removed))+ 0)]+ (when (pos? processed)+ (log/debug "Updated" processed "entities."))+ (- remaining removed processed))))+ batch-size+ entity-types)]+ (when (< remaining batch-size)+ (task.analysis-finding/trigger-analysis-finding-job!)+ (task.result-metadata/trigger-result-metadata-job!)+ true))))++(declare ^:private job)++(mu/defn- run-batch! :- [:maybe ms/PositiveInt]+ "Process a batch, returning when to run again: a second from now when it took any, or nil."+ []+ (when (update-dependencies!)+ 1))++(mu/defn- log-job-start :- :nil+ "Log the start of the job running in `ctx`, with how many triggers it has."+ [^JobExecutionContext ctx :- (ms/InstanceOfClass JobExecutionContext)]+ (let [scheduler (.getScheduler ctx)+ job-key (.getKey (.getJobDetail ctx))+ job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))+ active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]+ (log/debugf "Executing %s job. %s total triggers active." job-class active-triggers-count)))++(task/defjob UpdateDependencies+ "Update the dependency table."+ {:saved-class "metabase_enterprise.dependencies.task.backfill.BackfillDependencies"+ :concurrent? false}+ [ctx]+ (let [ctx ^JobExecutionContext ctx]+ (log-job-start ctx)+ (task.util/run-and-reschedule! (.getScheduler ctx) job)))++(def ^:private job+ {:job-type UpdateDependencies+ :job-key "metabase.task.dependency-backfill.job"+ :trigger-key "metabase.task.dependency-backfill.trigger"+ :job-name "Update Dependencies"+ :batch-size deps.settings/dependency-backfill-batch-size+ :delay-minutes deps.settings/dependency-backfill-delay-minutes+ :variance-minutes deps.settings/dependency-backfill-variance-minutes+ :run-batch! #'run-batch!})++(mu/defn trigger-dependency-job! :- :any+ "Trigger the UpdateDependencies job, unless a MB_DEPENDENCY_BACKFILL_BATCH_SIZE that isn't positive+ disables it."+ []+ (task.util/trigger-run! job))++(defmethod task/init! ::UpdateDependencies [_]+ (task.util/init-job! job))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.cljdeleted file mode 100644index d7b36c65ce3..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj+++ /dev/null@@ -1,97 +0,0 @@-(ns metabase-enterprise.dependencies.task.entity-check- (:require- [clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.triggers :as triggers]- [java-time.api :as t]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]- [metabase-enterprise.dependencies.settings :as deps.settings]- [metabase-enterprise.dependencies.task-util :as deps.task-util]- [metabase.premium-features.core :as premium-features]- [metabase.task.core :as task]- [metabase.util.log :as log]))--(set! *warn-on-reflection* true)--(defn- process-one-batch!- "Process one batch of entities. Returns true if the full batch was used."- []- (-> (reduce (fn [batch-size entity-type]- (if (< batch-size 1)- (reduced 0)- (let [processed (deps.findings/analyze-batch! entity-type batch-size)]- (when (pos? processed)- (log/info "Updated" processed "entities of type" entity-type))- (- batch-size processed))))- (deps.settings/dependency-entity-check-batch-size)- deps.findings/analyzable-entities)- (< 1)))--(defn- check-entities!- "Analyze stale entities in batches until none are left or a pass stops reducing their count."- []- (when (premium-features/has-feature? :dependencies)- (loop [prev-stale nil]- (process-one-batch!)- (let [stale (deps.analysis-finding/stale-entity-count)]- (when (and (pos? stale)- (or (nil? prev-stale) (< stale prev-stale)))- (recur stale))))))--(declare schedule-run!)--(defn- reschedule-after-run!- "Schedule the next run of the entity check job unless the `:dependencies` feature is definitively absent."- [scheduler]- (if (false? (premium-features/canonically-has-feature? :dependencies))- (log/debug "Not rescheduling job Dependency Entity Check: the dependencies feature is not enabled")- (schedule-run! scheduler- (deps.task-util/job-delay- (deps.settings/dependency-entity-check-delay-minutes)- (deps.settings/dependency-entity-check-variance-minutes)))))--(task/defjob DependencyEntityCheck- "Check all entities for validity"- {:saved-class "metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"- :concurrent? false}- [ctx]- (log/info "Executing DependencyEntityCheck job...")- (check-entities!)- (reschedule-after-run! (.getScheduler ctx)))--(def ^:private job-key "metabase.dependencies.task.entity-check.job")-(def ^:private trigger-key "metabase.dependencies.task.entity-check.trigger")--(defn- schedule-run!- "Schedule a run of the entity check job `delay-in-seconds` from now."- [scheduler delay-in-seconds]- (let [start-at (-> (t/instant)- (t/+ (t/duration delay-in-seconds :seconds))- java.util.Date/from)- trigger (triggers/build- (triggers/with-identity (triggers/key trigger-key))- (triggers/for-job job-key)- (triggers/start-at start-at))- job (jobs/build (jobs/of-type DependencyEntityCheck) (jobs/with-identity job-key))]- (log/info "Scheduling next run of job Dependency Entity Check at" start-at)- (task/schedule-task! scheduler job trigger)))--(defmethod task/init! ::DependencyEntityCheck [_]- (when-not (pos? (deps.settings/dependency-entity-check-batch-size))- (log/info "Dependency entity check batch size is not positive; the job will run but process nothing"))- (schedule-run!- (task/scheduler)- (deps.task-util/job-initial-delay- (deps.settings/dependency-entity-check-variance-minutes))))--(defn trigger-entity-check-job!- "Trigger the DependencyEntityCheck job to run after a brief delay, unless the batch size disables it.-- The 1-second delay ensures the calling transaction has committed before the job checks for stale entities. Entity- changes fire this, so leaving it ungated is what made a disabled job wake roughly once a second; the job's own- periodic schedule is left alone so it still resumes if the batch size becomes positive.-- Disable with MB_DEPENDENCY_ENTITY_CHECK_BATCH_SIZE=0"- []- (when (pos? (deps.settings/dependency-entity-check-batch-size))- (schedule-run! (task/scheduler) 1)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.cljnew file mode 100644index 00000000000..020ed205c41--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/result_metadata.clj@@ -0,0 +1,53 @@+(ns metabase-enterprise.dependencies.task.result-metadata+ "Implements a task that re-derives the result metadata of the MBQL cards depending on changed entities, see+ [[metabase-enterprise.dependencies.result-metadata]]."+ (:require+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase-enterprise.dependencies.task.util :as task.util]+ [metabase.premium-features.core :as premium-features]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms])+ (:import (org.quartz JobExecutionContext)))++(set! *warn-on-reflection* true)++(declare ^:private job)++(mu/defn- run-batch! :- [:maybe ms/PositiveInt]+ "Re-derive a batch of stale cards, returning when to run again: a second from now when it took any, or nil."+ []+ (when (premium-features/has-feature? :dependencies)+ (let [taken (deps.result-metadata/update-result-metadata! (deps.settings/dependency-result-metadata-batch-size))]+ (when (pos? taken)+ (log/debug "Processed" taken "entities for result metadata")+ 1))))++(task/defjob UpdateResultMetadata+ "Re-derive the result metadata of the MBQL cards depending on changed entities."+ {:saved-class "metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata"+ :concurrent? false}+ [ctx]+ (log/debug "Executing UpdateResultMetadata job...")+ (task.util/run-and-reschedule! (.getScheduler ^JobExecutionContext ctx) job))++(def ^:private job+ {:job-type UpdateResultMetadata+ :job-key "metabase.dependencies.task.result-metadata.job"+ :trigger-key "metabase.dependencies.task.result-metadata.trigger"+ :job-name "Update Result Metadata"+ :batch-size deps.settings/dependency-result-metadata-batch-size+ :delay-minutes deps.settings/dependency-result-metadata-delay-minutes+ :variance-minutes deps.settings/dependency-result-metadata-variance-minutes+ :run-batch! #'run-batch!})++(mu/defn trigger-result-metadata-job! :- :any+ "Trigger the UpdateResultMetadata job, unless a MB_DEPENDENCY_RESULT_METADATA_BATCH_SIZE that isn't positive+ disables it."+ []+ (task.util/trigger-run! job))++(defmethod task/init! ::UpdateResultMetadata [_]+ (task.util/init-job! job))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/util.cljnew file mode 100644index 00000000000..8d9b5ffc097--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/util.clj@@ -0,0 +1,115 @@+(ns metabase-enterprise.dependencies.task.util+ "The scheduling the dependencies module's self-rescheduling jobs share, each job described by a map of its+ `:job-type`, `:job-key`, `:trigger-key` and `:job-name`, the getters of its `:batch-size`, `:delay-minutes` and+ `:variance-minutes` settings, and `:run-batch!`, which processes a batch and returns how many seconds from now to+ run again, or nil to stop."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms])+ (:import+ (java.util Date)+ (org.quartz Scheduler)))++(set! *warn-on-reflection* true)++(def ^:private Job+ "A self-rescheduling job, as the namespace docstring describes it."+ [:map {:closed true}+ [:job-type [:fn {:error/message "a job class"} class?]]+ [:job-key :string]+ [:trigger-key :string]+ [:job-name :string]+ [:batch-size [:fn {:error/message "a setting getter"} ifn?]]+ [:delay-minutes [:fn {:error/message "a setting getter"} ifn?]]+ [:variance-minutes [:fn {:error/message "a setting getter"} ifn?]]+ [:run-batch! [:fn {:error/message "a function"} ifn?]]])++(def ^:private MaybeScheduler+ "A Quartz scheduler, or nil when none is running."+ [:maybe (ms/InstanceOfClass Scheduler)])++(mu/defn- job-delay :- ms/IntGreaterThanOrEqualToZero+ "A random delay in seconds between `delay - variance` and `delay + variance` minutes."+ [delay :- :int+ variance :- :int]+ (let [delay-duration (t/duration delay :minutes)+ variance-duration (t/duration variance :minutes)]+ (max 0 (.. delay-duration+ (minus variance-duration)+ (plusSeconds (rand-int (.. variance-duration+ (multipliedBy 2)+ toSeconds)))+ toSeconds))))++(mu/defn next-run-delay :- ms/IntGreaterThanOrEqualToZero+ "A random delay in seconds before `job` runs again after a failure, within its variance of its delay."+ [{:keys [delay-minutes variance-minutes]} :- Job]+ (job-delay (delay-minutes) (variance-minutes)))++(mu/defn- next-fire-time :- [:maybe (ms/InstanceOfClass Date)]+ "When the trigger with `trigger-key` fires next on `scheduler`, or nil when it is not pending."+ ^Date [^Scheduler scheduler :- MaybeScheduler+ trigger-key :- :string]+ (some-> scheduler (.getTrigger (triggers/key trigger-key)) .getNextFireTime))++(def ^:private schedule-lock+ "The lock that makes reading and replacing a job's pending trigger one step for this instance."+ (Object.))++(mu/defn schedule-run! :- :any+ "Schedule a run of `job` `delay-in-seconds` from now on `scheduler`, leaving a run already pending before then in+ place, or a second from now when that pending run is past due, since Quartz can leave a trigger past due for+ minutes."+ [scheduler :- MaybeScheduler+ {:keys [job-type job-key trigger-key job-name]} :- Job+ delay-in-seconds :- ms/IntGreaterThanOrEqualToZero]+ (locking schedule-lock+ (let [now (t/instant)+ start-at (t/+ now (t/duration delay-in-seconds :seconds))+ pending (some-> (next-fire-time scheduler trigger-key) .toInstant)]+ (when-not (and pending (t/after? pending now) (not (t/after? pending start-at)))+ (let [start-at (Date/from (if (or (nil? pending) (t/after? pending start-at))+ start-at+ (t/+ now (t/duration 1 :seconds))))+ trigger (triggers/build+ (triggers/with-identity (triggers/key trigger-key))+ (triggers/for-job job-key)+ (triggers/start-at start-at))+ job (jobs/build (jobs/of-type job-type) (jobs/with-identity job-key))]+ (log/debug "Scheduling next run of job" job-name "at" start-at)+ (task/schedule-task! scheduler job trigger))))))++(mu/defn run-and-reschedule! :- :any+ "Run a batch of `job` unless a batch size that isn't positive disables it, then schedule its next run when the batch+ asks for one, and after [[next-run-delay]] when it throws, before rethrowing, so a failure doesn't stop it until the+ next write."+ [scheduler :- MaybeScheduler+ {:keys [run-batch! job-name batch-size] :as job} :- Job]+ (if (pos? (batch-size))+ (try+ (if-some [delay-in-seconds (run-batch!)]+ (schedule-run! scheduler job delay-in-seconds)+ (log/debug "Job" job-name "has nothing more to do, stopping."))+ (catch Throwable e+ (schedule-run! scheduler job (next-run-delay job))+ (throw e)))+ (log/debug "Job" job-name "has a batch size that isn't positive, stopping.")))++(mu/defn trigger-run! :- :any+ "Schedule a run of `job` a second from now, unless a batch size that isn't positive disables it."+ [{:keys [batch-size] :as job} :- Job]+ (when (pos? (batch-size))+ (schedule-run! (task/scheduler) job 1)))++(mu/defn init-job! :- :any+ "Schedule the first run of `job` after startup, at a random time within its variance, unless a batch size that isn't+ positive disables it."+ [{:keys [batch-size variance-minutes job-name] :as job} :- Job]+ (if (pos? (batch-size))+ (schedule-run! (task/scheduler) job (rand-int (.toSeconds (t/duration (variance-minutes) :minutes))))+ (log/info "The batch size of job" job-name "is not positive; the job is disabled")))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.cljdeleted file mode 100644index 8efa5ffb6e5..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj+++ /dev/null@@ -1,27 +0,0 @@-(ns metabase-enterprise.dependencies.task-util- (:require- [java-time.api :as t]))--(set! *warn-on-reflection* true)--(defn job-delay- "Generates a random delay in seconds based on `delay` and `variance`.-- Both `delay` and `variance` should be in minutes, and the result will vary between `delay-variance` and- `delay+variance`."- [delay variance]- (let [delay-duration (t/duration delay :minutes)- variance-duration (t/duration variance :minutes)]- (max 0 (.. delay-duration- (minus variance-duration)- (plusSeconds (rand-int (.. variance-duration- (multipliedBy 2)- toSeconds)))- toSeconds))))--(defn job-initial-delay- "Generates a random delay in seconds based on `variance`.-- `variance` is in minutes and the result will vary between 0 and `variance`"- [variance]- (rand-int (.toSeconds (t/duration variance :minutes))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex fcbaa74296f..30c26ef859b 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -9,7 +9,6 @@[metabase-enterprise.serialization.v2.models :as serdes.models][metabase.app-db.core :as mdb][metabase.app-db.transient-error :as transient-error]- [metabase.dependencies.hooks :as dependencies.hooks][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.search.core :as search]@@ -288,20 +287,17 @@:files file-names}(first ingest-errors)))))(log/infof "Starting deserialization, total %s documents" (count contents))- (try- (reduce (fn [ctx item]- (try- (load-one! ctx item)- (catch Exception e- (when-not continue-on-error- (throw e))- ;; eschew big and scary stacktrace- (log/warnf (u/strip-error e "Skipping deserialization error"))- (update ctx :errors conj e))))- ctx- contents)- (finally- (dependencies.hooks/after-serdes-load))))+ (reduce (fn [ctx item]+ (try+ (load-one! ctx item)+ (catch Exception e+ (when-not continue-on-error+ (throw e))+ ;; eschew big and scary stacktrace+ (log/warnf (u/strip-error e "Skipping deserialization error"))+ (update ctx :errors conj e))))+ ctx+ contents))(when reindex?;; Reindex after all entities are loaded. Individual entity commits may have produced stale;; search index entries; this ensures the index reflects the final state.diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 92a174293f1..ca6dd888b74 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -76,6 +76,7 @@"AiUsageLog""AnalysisFinding""AnalysisFindingError"+ "AnalysisFindingStatus""ApiKey""ApiKeyUsageLog""ApplicationPermissionsRevision"@@ -158,6 +159,7 @@"ReplacementRun""RemoteSyncObject""RemoteSyncTask"+ "ResultMetadataStatus""Revision""Sandbox""SearchIndexMetadata"diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.cljindex ef97f08d808..94729b701b9 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj@@ -3,7 +3,6 @@[clojure.test :refer :all][metabase-enterprise.data-apps.group-access :as group-access][metabase-enterprise.dependencies.test-util :as deps.test]- [metabase.dependencies.hooks :as dependencies.hooks][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.permissions-rest.data-permissions.graph :as data-perms.graph]@@ -50,7 +49,7 @@:model/QueryAction _ {:action_id action-id:database_id (mt/id):dataset_query (lib/native-query mp "DELETE FROM CHECKINS WHERE ID = 1")}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)(group-access/add-groups! app [(:id group)])(is (=? [{:group_id (:id group)@@ -72,48 +71,29 @@:model/PermissionsGroup group {}](perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)(group-access/add-groups! app [(:id group)])- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}](group-access/permission-warnings app)))))))))))(deftest group-permission-warnings-follow-snippet-edit-test(testing "after a snippet edit and the jobs, the warnings list the tables of the snippet's new SQL"- (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}- (mt/with-no-data-perms-for-all-users!- (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]- (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}- :model/DataApp {collection-id :resource_collection_id :as app}- {:name "birds" :display_name "Birds" :bundle_path "birds.js"}- :model/Card _ {:collection_id collection-id- :dataset_query (lib/native-query (mt/metadata-provider)- "SELECT * FROM {{snippet: birds source}}")}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])- (deps.test/synchronously-run-backfill!)- (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})- (deps.test/synchronously-run-backfill!)- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}]- (group-access/permission-warnings app)))))))))--(deftest group-permission-warnings-include-table-synced-after-analysis-test- (testing "a table that sync finds after the app's card was analyzed enters the warnings once the jobs run"- (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}- (mt/with-no-data-perms-for-all-users!- (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]- (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}- {:name "birds" :display_name "Birds" :bundle_path "birds.js"}- :model/Card _ {:collection_id collection-id- :dataset_query (lib/native-query (mt/metadata-provider)- "SELECT * FROM PUBLIC.BIRDS_LATE")}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])- (deps.test/synchronously-run-backfill!)- (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]- (dependencies.hooks/after-database-sync (mt/id))- (deps.test/synchronously-run-backfill!)- (is (=? [{:group_id (:id group) :missing_tables [{:id late-id}]}]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}+ :model/DataApp {collection-id :resource_collection_id :as app}+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card _ {:collection_id collection-id+ :dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT * FROM {{snippet: birds source}}")}+ :model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (deps.test/synchronously-run-dependency-job!)+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})+ (deps.test/synchronously-run-dependency-job!)+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}](group-access/permission-warnings app))))))))))(deftest group-permission-warnings-exclude-inactive-tables-testdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.cljnew file mode 100644index 00000000000..873aef424e2--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/analysis_finding_test.clj@@ -0,0 +1,584 @@+(ns metabase-enterprise.dependencies.analysis-finding-test+ (:require+ [clojure.test :refer :all]+ [medley.core :as m]+ [metabase-enterprise.dependencies.analysis :as deps.analysis]+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]+ [metabase-enterprise.dependencies.hooks :as deps.hooks]+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.models.analysis-finding-error :as models.analysis-finding-error]+ [metabase-enterprise.dependencies.test-util :as deps.test]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]+ [metabase.query-processor :as qp]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(defn- run-jobs!+ "Run the dependency job, whose current dependencies the analysis job waits for, then the analysis job, until each has+ nothing left to do."+ []+ (deps.test/synchronously-run-dependency-job!)+ (mt/with-premium-features #{:dependencies}+ (deps.test/run-analysis-finding-job!)))++(defn- backfill-all-entity-analyses! []+ (run-jobs!))++(defn- status [entity-type entity-id]+ (t2/select-one :model/AnalysisFindingStatus :entity_type entity-type :entity_id entity-id))++(defn- analyzed-ids!+ "The IDs of the cards `thunk` analyzes, in order."+ [thunk]+ (let [analyzed (atom [])+ upsert! (mt/original-fn #'models.analysis-finding/upsert-analysis!)]+ (mt/with-dynamic-fn-redefs [models.analysis-finding/upsert-analysis!+ (fn [entity-type entity-id & args]+ (when (= :card entity-type)+ (swap! analyzed conj entity-id))+ (apply upsert! entity-type entity-id args))]+ (thunk))+ @analyzed))++(deftest ^:synchronized can-analyze-entity-batches-test+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products-id (mt/id :products)+ orders-id (mt/id :orders)+ products (lib.metadata/table mp products-id)+ orders (lib.metadata/table mp orders-id)]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]+ (run-jobs!)+ (is (= #{models.analysis-finding/*current-analysis-finding-version*}+ (t2/select-fn-set :analysis_version+ :model/AnalysisFindingStatus+ :entity_id [:in [card-id other-card-id]]+ :entity_type :card)))))))++(deftest ^:synchronized does-not-repeatedly-analyze-entities-test+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products-id (mt/id :products)+ orders-id (mt/id :orders)+ products (lib.metadata/table mp products-id)+ orders (lib.metadata/table mp orders-id)]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]+ (is (= #{card-id other-card-id} (set (filter #{card-id other-card-id} (analyzed-ids! run-jobs!)))))+ (is (empty? (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))))))++(deftest ^:synchronized re-analyze-entities-when-analysis-version-bumped-test+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]+ (run-jobs!)+ (is (empty? (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]+ (is (= #{card-id other-card-id} (set (filter #{card-id other-card-id} (analyzed-ids! run-jobs!))))))))))++(deftest ^:synchronized does-analyze-native-entities-test+ (testing "failed analysis result is appropriately stored in the appdb"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]+ (run-jobs!)+ (is (= models.analysis-finding/*current-analysis-finding-version*+ (:analysis_version (status :card card-id))))+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_id card-id+ :entity_type :card))))))))++(deftest ^:synchronized does-report-errors-for-missing-refs-in-fields-test+ (testing "missing (not inactive) field refs in :fields are reported as findings (GHY-3157)"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ base (lib/query mp orders)+ cols (vec (take 5 (lib/returned-columns base orders)))+ bad-col (-> (first cols)+ lib/ref+ (assoc 2 "bad_column"))+ query (lib/with-fields base (conj cols bad-col))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]+ (run-jobs!)+ (is (= models.analysis-finding/*current-analysis-finding-version*+ (:analysis_version (status :card card-id))))+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_id card-id+ :entity_type :card))))))))++;; TODO: (bshepherdson, 2026-02-05) Add a test like does-not-report-errors-in-removable-refs-test-1-stage-fields for+;; join clause :fields as well. See QUE-3081 and QUE-3044.++(deftest ^:parallel reports-missing-field-in-downstream-card-fields-test+ (testing "Q2 based on Q1 with :fields referencing a column Q1 no longer returns should be reported (GHY-3157)"+ (let [mp meta/metadata-provider+ q1 (lib/query mp (meta/table-metadata :orders))+ q1-cols (lib/returned-columns q1)+ removed-col (m/find-first #(= (:name %) "ID") q1-cols)+ ;; Card 101 = Q1 with ID removed from result-metadata (simulates Q1 dropping that field)+ mp (lib.tu/metadata-provider-with-card-from-query+ mp 101 q1+ {:result-metadata (vec (remove #(= (:name %) "ID") q1-cols))})+ ;; Q2: query based on card 101, with :fields including the now-missing ID+ q2-base (lib/query mp {:lib/type :metadata/card :id 101})+ q2-cols (lib/returned-columns q2-base)+ q2 (lib/with-fields q2-base (conj (vec q2-cols) (lib/ref removed-col)))+ ;; Card 102 = Q2 (provide result-metadata explicitly since q2 has a bad ref)+ mp (lib.tu/metadata-provider-with-card-from-query+ mp 102 q2+ {:result-metadata (vec q2-cols)})+ results (deps.analysis/check-entity mp :card 102)]+ (is (seq results)+ "Q2 should have findings for the missing ID column from Q1"))))++(deftest ^:synchronized analyze-batch-picks-up-missing-analyses-test+ (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]+ (testing "card has no analysis finding initially"+ (is (not (t2/exists? :model/AnalysisFinding+ :entity_type :card+ :entity_id card-id))))+ (testing "the jobs create the analysis of the card"+ (run-jobs!))+ (testing "card now has an analysis finding"+ (is (t2/exists? :model/AnalysisFinding+ :entity_type :card+ :entity_id card-id))))))))++(deftest ^:synchronized failed-analysis-waits-for-change-test+ (testing "an entity whose analysis keeps throwing records a failed finding, and is given up on until it changes"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFindingError]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (fn [_] (throw (ex-info "Broken" {})))}+ run-jobs!)+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_type :card :entity_id card-id)))+ (is (empty? (filter #{card-id} (analyzed-ids! run-jobs!))))))))))++(deftest ^:synchronized analysis-re-analyzes-changed-entity-and-direct-dependents-test+ (testing "after a card's query changes, the analysis finding job re-analyzes it and the cards directly depending on+ it, and nothing for a question's rename"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {gp-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {p-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}+ :model/Card {c-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}+ :model/Dependency _ {:from_entity_type :card :from_entity_id p-id+ :to_entity_type :card :to_entity_id gp-id}+ :model/Dependency _ {:from_entity_type :card :from_entity_id c-id+ :to_entity_type :card :to_entity_id p-id}]+ (run-jobs!)+ (t2/update! :model/Card gp-id {:name "Renamed"})+ (is (empty? (filter #{gp-id p-id c-id} (analyzed-ids! run-jobs!))))+ (t2/update! :model/Card gp-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})+ (is (= #{gp-id p-id}+ (set (filter #{gp-id p-id c-id} (analyzed-ids! run-jobs!)))))))))))++(deftest ^:synchronized analysis-follows-table-field-write-test+ (testing "a write of a field of a table re-analyzes the entities depending on the table"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :products)))}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :products)}]+ (run-jobs!)+ (is (not (deps.test/analysis-finding-pending? :card card-id)))+ (mt/with-temp-vals-in-db :model/Field (mt/id :products :title) {:description "Changed"}+ (is (deps.test/analysis-finding-pending? :table (mt/id :products)))+ (is (some #{card-id} (analyzed-ids! run-jobs!))))))))))++(deftest ^:synchronized cyclic-dependency-drain-terminates-test+ (testing "the analysis finding job terminates on a cyclic dependency graph"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {a-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {b-id :id} {:dataset_query (lib/query mp products)}+ ;; cycle: a depends on b, and b depends on a+ :model/Dependency _ {:from_entity_type :card :from_entity_id a-id+ :to_entity_type :card :to_entity_id b-id}+ :model/Dependency _ {:from_entity_type :card :from_entity_id b-id+ :to_entity_type :card :to_entity_id a-id}]+ (let [done (future (run-jobs!))]+ (is (not= ::timeout (deref done 30000 ::timeout))+ "the analysis finding job must terminate on a cyclic graph")+ (is (not (deps.test/run-analysis-finding-job!)) "and leave nothing to do")))))))++(deftest ^:synchronized python-transform-records-clean-finding-test+ (testing "GHY-3584: a Python transform has no query, so upsert-analysis! resolves its database from source_database_id and records a clean finding"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFindingError]+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :python+ :source-database (mt/id)+ :source-tables []+ :body "def transform():\n pass"}}]+ (lib-be/with-metadata-provider-cache+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Transform transform-id)))+ (is (=? {:result true}+ (t2/select-one :model/AnalysisFinding :entity_type :transform :entity_id transform-id)))+ (is (empty? (models.analysis-finding-error/errors-for-entity :transform transform-id))))))))++(deftest ^:synchronized unanalyzable-entity-records-error-test+ (testing "an entity whose database can't be resolved records an error finding instead of no-oping forever"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFindingError]+ (mt/with-temp [:model/Card card {:dataset_query (mt/native-query {:query "SELECT 1"})+ :database_id (mt/id)}]+ (let [cid (:id card)]+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (constantly nil)}+ (fn []+ (lib-be/with-metadata-provider-cache+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))))+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_type :card :entity_id cid)))+ (is (seq (models.analysis-finding-error/errors-for-entity :card cid))+ "an error finding is recorded explaining why the entity couldn't be analyzed")))))))++(deftest ^:synchronized unanalyzable-entity-rechecks-when-database-resolvable-again-test+ (testing "after a no-database error, the entity is analyzed normally once its database resolves again"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFindingError]+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp products)}]+ (let [cid (:id card)]+ (with-redefs-fn {#'deps.analysis-finding/entity-db-id (constantly nil)}+ (fn []+ (lib-be/with-metadata-provider-cache+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))))+ (is (seq (models.analysis-finding-error/errors-for-entity :card cid)))+ (lib-be/with-metadata-provider-cache+ (deps.analysis-finding/upsert-analysis! (t2/instance :model/Card card)))+ (is (empty? (models.analysis-finding-error/errors-for-entity :card cid))+ "re-analyzed normally: the database error is replaced by a clean result"))))))))++(deftest ^:synchronized analyze-batch-shares-one-metadata-provider-test+ (testing "Every entity in an analysis batch must share one MetadataProvider per database. Reading an entity attaches+ a provider to its query, so selecting the batch outside the cache gives each entity a private one. Nothing+ reads through those providers today -- upsert-analysis! builds its own from the database id and passes it+ explicitly -- so this is a latent trap rather than a live leak: the moment an entity is analyzed through its+ own query, a batch of 500 retains 500 providers' worth of fetched metadata. upsert-analysis! already throws+ when no cache is bound, so the cache is meant to cover this work; only the select sat outside it."+ (backfill-all-entity-analyses!)+ (let [providers (atom [])+ native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}+ :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]+ (let [wanted #{card1-id card2-id}+ upsert-analysis! (mt/original-fn #'deps.analysis-finding/upsert-analysis!)]+ (mt/with-dynamic-fn-redefs [deps.analysis-finding/upsert-analysis!+ (fn [entity & args]+ (when (wanted (:id entity))+ (swap! providers conj (:lib/metadata (:dataset_query entity))))+ (apply upsert-analysis! entity args))]+ (run-jobs!)))+ (testing "both cards were selected, so the assertion below is meaningful"+ (is (= 2 (count @providers))))+ (testing "and they carry the same provider instance"+ ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be+ ;; satisfied by two separate caches and would not detect the regression.+ (is (every? #(identical? (first @providers) %) @providers))))))))++(defn- analyzed-by-job!+ "The entities, as `[entity-type entity-id]`, that running the analysis finding job now analyzes, after the dependency+ job it waits for."+ []+ (deps.test/synchronously-run-dependency-job!)+ (let [analyzed (atom #{})+ upsert! (mt/original-fn #'models.analysis-finding/upsert-analysis!)]+ (mt/with-dynamic-fn-redefs [models.analysis-finding/upsert-analysis!+ (fn [entity-type entity-id & args]+ (swap! analyzed conj [entity-type entity-id])+ (apply upsert! entity-type entity-id args))]+ (deps.test/run-analysis-finding-job!))+ @analyzed))++(defn- analyzed-among!+ "Which of the entities `nodes`, `[entity-type entity-id]`, running the analysis finding job now analyzes."+ [& nodes]+ (into #{} (filter (set nodes)) (analyzed-by-job!)))++(defn- assert-has-analyses+ "Asserts that a given set of entities have the appropriate analysis version.++ Format is `{entity-type {entity-id analysis-version}}`.++ Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version.++ Use nil for analysis-version if you want to check that no analysis exists."+ [spec]+ (doseq [[entity-type ids-and-versions] spec]+ (let [analyses-map (->> (t2/select [:model/AnalysisFindingStatus :entity_id :analysis_version]+ :entity_type entity-type)+ (into {} (map (juxt :entity_id :analysis_version))))]+ (doseq [[id expected-version] ids-and-versions]+ (testing (str "Checking " entity-type " " id)+ (if (= expected-version -1)+ (is (contains? analyses-map id))+ (is (= expected-version (analyses-map id)))))))))++(deftest ^:synchronized card-update-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Card updates should mark the card stale and trigger the background job"+ (let [products-id (mt/id :products)+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id child-card-id+ :to_entity_type :card+ :to_entity_id parent-card-id}]+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})+ (assert-has-analyses+ {:card {parent-card-id nil}})+ (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFindingStatus+ :entity_type :card+ :entity_id child-card-id))+ "an update writes no analysis record")))))))++(deftest ^:synchronized card-update-updates-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "changing a card's query has the job re-analyze it and its dependents"+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id child-card-id+ :to_entity_type :card+ :to_entity_id parent-card-id}]+ (analyzed-by-job!)+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})+ (deps.test/synchronously-run-dependency-job!)+ (is (= #{[:card parent-card-id] [:card child-card-id]}+ (analyzed-among! [:card parent-card-id] [:card child-card-id])))))))))++(deftest ^:synchronized card-update-triggers-native-cards-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a native card's update with new run metadata has the job re-analyze it and its direct dependents"+ (mt/with-temp [:model/Card {parent-id :id} {:dataset_query (lib/native-query mp "select * from products")}+ :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-id+ :to_entity_type :card :to_entity_id parent-id}]+ (analyzed-by-job!)+ (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")+ :result_metadata (get-in (qp/process-query (lib/native-query mp "select * from orders"))+ [:data :results_metadata :columns])})+ (deps.test/synchronously-run-dependency-job!)+ (is (= #{[:card parent-id] [:card child-id]}+ (analyzed-among! [:card parent-id] [:card child-id]))))))))++(deftest ^:synchronized card-update-analyzes-dependents-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a card update has the job re-analyze the card and its dependents, transforms included"+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}]+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/query mp (lib.metadata/card+ (mt/metadata-provider)+ parent-card-id))}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]+ (analyzed-by-job!)+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})+ (is (= #{[:card parent-card-id] [:transform transform-id]}+ (analyzed-among! [:card parent-card-id] [:transform transform-id] [:card other-card-id]))))))))))++(deftest ^:synchronized transform-update-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Transform update does not error when no analyses exist yet"+ (let [products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (is (= 1 (t2/update! :model/Transform transform-id+ {:source {:type :query+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))++(deftest ^:synchronized transform-update-updates-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a transform update has the job re-analyze it"+ (let [products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (analyzed-by-job!)+ (t2/update! :model/Transform transform-id {:source {:type :query+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})+ (is (= #{[:transform transform-id]} (analyzed-among! [:transform transform-id])))))))))++(deftest ^:synchronized transform-update-triggers-native-transforms-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Native transform updates do not error"+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/native-query mp "select * from products")}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (is (= 1 (t2/update! :model/Transform transform-id+ {:source {:type :query+ :query (lib/native-query mp "select * from orders")}}))))))))++(deftest ^:synchronized transform-run-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Transform run completes without error when no analyses exist"+ (let [products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public" :name "sample" :type :table}}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :transform :to_entity_id transform-id}]+ (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))++(deftest ^:synchronized transform-run-without-table-analyzes-nothing-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a transform run re-analyzes nothing by itself, since the sync of its output table marks what reads that table"+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public" :name "sample" :type :table}}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :transform :to_entity_id transform-id}]+ (analyzed-by-job!)+ (deps.hooks/after-transform-run transform-id nil)+ (is (= #{}+ (analyzed-among! [:card card-id] [:transform transform-id] [:card other-card-id])))))))))++(deftest ^:synchronized segment-update-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [_]+ (testing "Segment update does not error when no analyses exist yet"+ (let [products-id (mt/id :products)+ price-field-id (mt/id :products :price)]+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}]+ (is (= 1 (t2/update! :model/Segment segment-id+ {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))++(deftest ^:synchronized segment-update-analyzes-segment-and-dependents-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a segment update has the job re-analyze the segment and the cards using it"+ (let [products-id (mt/id :products)+ price-field-id (mt/id :products :price)+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/filter (lib/query mp products)+ (lib.metadata/segment (mt/metadata-provider)+ segment-id))}]+ (analyzed-by-job!)+ (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})+ (is (= #{[:segment segment-id] [:card card-id]}+ (analyzed-among! [:segment segment-id] [:card card-id]))))))))))++(deftest ^:synchronized table-metadata-update-triggers-dependent-analysis-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a table metadata update has the job re-analyze the table's dependents, not unrelated cards"+ (let [products-id (mt/id :products)+ orders (lib.metadata/table mp (mt/id :orders))+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}]+ (analyzed-by-job!)+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]+ (is (= #{[:card card-id]} (analyzed-among! [:card card-id] [:card other-card-id]))))))))))++(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Table metadata update does not error when no findings exist yet"+ (let [products-id (mt/id :products)+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}]+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]+ (assert-has-analyses {:card {card-id nil}}))))))))++(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "a field metadata update has the job re-analyze the dependents of the field's table"+ (let [products-id (mt/id :products)+ orders (lib.metadata/table mp (mt/id :orders))+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}]+ (analyzed-by-job!)+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category)+ :display_name "Category renamed"}]+ (is (= #{[:card card-id]} (analyzed-among! [:card card-id] [:card other-card-id]))))))))))++(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test+ (deps.test/run-with-dependencies-setup!+ (fn [mp]+ (testing "Field metadata update does not error when no findings exist yet"+ (let [products-id (mt/id :products)+ products (lib.metadata/table mp products-id)]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}]+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]+ (assert-has-analyses {:card {card-id nil}}))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex 8bd8650d9e0..c3e6b382dd6 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -2,10 +2,9 @@(:require[clojure.test :refer :all][medley.core :as m]+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.api :as deps.api]- [metabase-enterprise.dependencies.findings :as dependencies.findings][metabase-enterprise.dependencies.hooks :as deps.hooks]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase-enterprise.dependencies.test-util :as deps.test][metabase.collections.models.collection :as collection][metabase.collections.test-utils :refer [personal-collection-id]]@@ -128,7 +127,7 @@"Run analysis for a specific card to detect broken references.Must be called within lib-be/with-metadata-provider-cache."[card-id]- (dependencies.findings/upsert-analysis! (t2/select-one :model/Card :id card-id)))+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Card :id card-id)))(deftest graph-test(testing "GET /api/ee/dependencies/graph"@@ -137,7 +136,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}](let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user){card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph" :id card-id-2 :type "card")creator {:email "[email protected]":id (:id user)}]@@ -198,7 +197,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}](let [_card-1 (card/create-card! (basic-card "Card 1" :orders) user)_card-2 (card/create-card! (basic-card "Card 2" :orders) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph":id (mt/id :orders):type "table")]@@ -247,7 +246,7 @@:target {:type "table" :name (mt/random-name)}}](events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})(events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id products-id :type "table")table-node (first (filter #(= (:id %) products-id) (:nodes response)))]@@ -267,7 +266,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}](let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user){card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id card-id-1:type "card"@@ -307,7 +306,7 @@:size_x 4:size_y 4})(events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "single dependent-types value (backward compatibility)"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id card-id-1@@ -339,7 +338,7 @@(let [{dependency-card-id :id :as dependency-card} (card/create-card! (basic-card "Base card - cardtypes") user)_question-card (card/create-card! (assoc (wrap-card dependency-card) :name "Question card") user)_model-card (card/create-card! (assoc (wrap-card dependency-card) :name "Model card" :type :model) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "single dependent-card-types value"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id dependency-card-id@@ -381,7 +380,7 @@dependent-card (card/create-card! (wrap-card base-card) user)](card/update-card! {:card-before-update base-card:card-updates {:archived true}})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph":id (:id dependent-card):type "card")@@ -403,7 +402,7 @@dependent-card (card/create-card! (wrap-card base-card) user)](card/update-card! {:card-before-update base-card:card-updates {:archived true}})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph":id (:id dependent-card):type "card")@@ -426,7 +425,7 @@dependent-card (card/create-card! (wrap-card base-card) user)](card/update-card! {:card-before-update dependent-card:card-updates {:archived true}})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id (:id base-card):type "card"@@ -450,7 +449,7 @@;; Simulate the transform having been run: link it to its (now inactive) target table(t2/update! :model/Transform transform-id {:target_table_id table-id})(deps.hooks/after-transform-run transform-id table-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "GET /graph/dependents returns the inactive table as a child"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":id transform-id@@ -479,7 +478,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card)(let [next-card (create-dependent-card-on-model! user model-card "Another Dependent Card - brokentest" {:table :products})]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id next-card))(run-analysis-for-card! (:id dependent-card))(let [response2 (mt/user-http-request :crowberto :get 200 (str "ee/dependencies/graph/dependents?broken=true&type=card&id=" (:id model-card)))]@@ -503,8 +502,8 @@:table "ORDERS":table_id (mt/id :orders)}]:body "def transform(orders):\n return orders"}}]- (deps.test/synchronously-run-backfill!)- (#'task.entity-check/check-entities!)+ (deps.test/synchronously-run-dependency-job!)+ (deps.test/run-analysis-finding-job!)(let [dependent-ids (fn [broken?](set (map :id (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":type "table"@@ -524,7 +523,7 @@:model/User user {:email "[email protected]"}](mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "Returns 403 when user lacks read permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 "ee/dependencies/graph"@@ -544,7 +543,7 @@:model/User user {:email "[email protected]"}](mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [{card-id :id} (card/create-card! (assoc (basic-card) :collection_id (:id collection)) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "Returns 403 when user lacks read permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 "ee/dependencies/graph/dependents"@@ -576,7 +575,7 @@top-card (card/create-card! (assoc (wrap-two-cards readable-base unreadable-base):collection_id (:id readable-collection))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees complete upstream graph through readable path"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -617,7 +616,7 @@unreadable-dependent (card/create-card! (assoc (wrap-card base-card):collection_id (:id unreadable-collection))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only readable dependents"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"@@ -649,7 +648,7 @@end-card (card/create-card! (assoc (wrap-two-cards unreadable-middle readable-alternate):collection_id (:id readable-collection))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "Diamond pattern: complete upstream graph via readable path"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -694,7 +693,7 @@top-card (card/create-card! (assoc (wrap-card unreadable-base):collection_id (:id readable-collection))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only the top card in the graph"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -742,7 +741,7 @@:visualization_settings {}:collection_id (:id unreadable-collection)}user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only readable cards as dependents of the snippet"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"@@ -786,7 +785,7 @@(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/create-queries :no)(let [card (card/create-card! (assoc (basic-card "Card on orders"):collection_id (:id readable-collection)) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only the card, table is filtered out"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -813,7 +812,7 @@dashboard-card (card/create-card! (assoc (wrap-card base-card):dashboard_id (:id dashboard))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "GET /api/ee/dependencies/graph returns dashboard with :id and :name"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id (:id dashboard-card)@@ -846,7 +845,7 @@document-card (card/create-card! (assoc (wrap-card base-card):document_id (:id document))user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "GET /api/ee/dependencies/graph returns document with :id and :name"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id (:id document-card)@@ -879,7 +878,7 @@:table_id products-id:definition {:filter [:> [:field price-field-id nil] 50]}}](events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id segment-id:type "segment")@@ -908,7 +907,7 @@query-with-segment (-> (lib/query mp products)(lib/filter (lib.metadata/segment mp segment-id)))card (card/create-card! (card-with-query "Card using segment" query-with-segment) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":id segment-id:type "segment"@@ -960,7 +959,7 @@:card-id model-id}}))}](mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [dependents (fn [user](->> (mt/user-http-request user :get 200 "ee/dependencies/graph/dependents":id model-id@@ -988,7 +987,7 @@:definition (-> (lib/query mp products)(lib/aggregate (lib/sum price)))}](events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-id:type "measure")@@ -1014,7 +1013,7 @@:definition (-> (lib/query mp products)(lib/aggregate (lib/sum price)))}](events/publish-event! :event/measure-create {:object measure-a :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [mp' (mt/metadata-provider)](mt/with-temp [:model/Measure {measure-b-id :id :as measure-b} {:name "Measure B":table_id products-id@@ -1022,7 +1021,7 @@(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)(lib/sum rating))))}](events/publish-event! :event/measure-create {:object measure-b :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-b-id:type "measure")@@ -1058,7 +1057,7 @@query-with-measure (-> (lib/query mp' products)(lib/aggregate (lib.metadata/measure mp' measure-id)))card (card/create-card! (card-with-query "Card using measure" query-with-measure) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":id measure-id:type "measure"@@ -1116,7 +1115,7 @@;; Archive measure B (the middle of the chain)(mt/user-http-request :crowberto :put 200 (str "measure/" measure-b-id){:archived true :revision_message "Archive middle measure"})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "after archiving measure B, it and measure A are excluded (chain broken)"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-c-id@@ -1188,7 +1187,7 @@(lib/query mp))}](events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id referenced-card-id) :user-id (mt/user->id :crowberto)})(events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id unreffed-card-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=unreftest")](is (=? {:data [{:id unreffed-card-id:type "card"@@ -1205,7 +1204,7 @@:type :question:dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}](events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=unreftest")](is (=? {:data [{:id unreffed-table-id:type "table"@@ -1235,7 +1234,7 @@;; so the table→transform dep is visible in the dependency graph.(t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})(deps.hooks/after-transform-run referenced-transform-id referenced-table-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")](is (=? {:data [{:id unreffed-transform-id:type "transform"@@ -1261,7 +1260,7 @@:type :question:dataset_query native-query}](events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=unreftest")](is (=? {:data [{:id unreffed-snippet-id:type "snippet"@@ -1281,7 +1280,7 @@"model" "dashboard"}}]}]}:content_type "application/json+vnd.prose-mirror"}](events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id document-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=unreftest")](is (=? {:data [{:id unreffed-dashboard-id:type "dashboard"@@ -1300,7 +1299,7 @@"model" "document"}}]}]}:content_type "application/json+vnd.prose-mirror"}](events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id unreffed-document-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=unreftest")](is (=? {:data [{:id unreffed-document-id:type "document"@@ -1319,7 +1318,7 @@:model/Sandbox {sandbox-id :id} {:group_id group-id:table_id (mt/id :products):card_id sandbox-card-id}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=sandbox")](is (=? {:data [{:id sandbox-id:type "sandbox"@@ -1337,7 +1336,7 @@:model/Card {unreffed-metric-id :id} {:name "B - Unreferenced Metric - cardtype":type :metric:dataset_query (lib/query mp products)}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "filtering by model only"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=model&query=cardtype")](is (=? {:data [{:id unreffed-model-id@@ -1376,7 +1375,7 @@:type :question:archived true:dataset_query (lib/query mp products)}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=archivedtest")card-ids (set (map :id (:data response)))](is (contains? card-ids unreffed-card-id))@@ -1388,7 +1387,7 @@(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}:model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest":archived true}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")dashboard-ids (set (map :id (:data response)))](is (contains? dashboard-ids unreffed-dashboard-id))@@ -1400,7 +1399,7 @@(mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}:model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest":archived true}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")document-ids (set (map :id (:data response)))](is (contains? document-ids unreffed-document-id))@@ -1414,7 +1413,7 @@:model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest":content "WHERE ID > 20":archived true}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")snippet-ids (set (map :id (:data response)))](is (contains? snippet-ids unreffed-snippet-id))@@ -1435,7 +1434,7 @@:model/Dashboard {regular-dashboard-id :id} {:name "Regular Dashboard - intcontent"}:model/Dashboard {internal-dashboard-id :id} {:name "Internal Dashboard - intcontent":creator_id config/internal-mb-user-id}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "internal-user cards are filtered out"(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced?types=card&query=intcontent")@@ -1463,7 +1462,7 @@(lib-be/with-metadata-provider-cache(break-model-card! (t2/select-one :model/Card :id (:id internal-model))))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card)))(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/breaking?types=card&query=intbreaking")@@ -1485,7 +1484,7 @@:archived true}](events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})(events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")segment-ids (set (map :id (:data response)))](is (contains? segment-ids unreffed-segment-id))@@ -1504,7 +1503,7 @@:db_id (mt/id):active true:visibility_type "hidden"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")table-ids (set (map :id (:data response)))](is (contains? table-ids active-table-id))@@ -1519,7 +1518,7 @@(mt/with-temp [:model/Table {table1-id :id} {:name "Table 1 - unreftest"}:model/Table {table2-id :id} {:name "Table 2 - unreftest"}:model/Table {table3-id :id} {:name "Table 3 - unreftest"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? {:data [{:id table1-id} {:id table2-id}]:total 3:offset 0@@ -1568,7 +1567,7 @@:archived true}](events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})(events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")measure-ids (set (map :id (:data response)))](is (contains? measure-ids unreffed-measure-id))@@ -1594,7 +1593,7 @@:model/Card {card-regular :id} {:name "Card Regular - personalcolltest":type :question:dataset_query (lib/query mp products)}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "include-personal-collections=false (default) excludes cards in personal collections"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=personalcolltest")card-ids (set (map :id (:data response)))]@@ -1616,7 +1615,7 @@:model/Dashboard {dash-in-personal :id} {:name "Dashboard in Personal - personalcolltest":collection_id (personal-collection-id user-id)}:model/Dashboard {dash-regular :id} {:name "Dashboard Regular - personalcolltest"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "include-personal-collections=false (default) excludes dashboards in personal collections"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=personalcolltest")dashboard-ids (set (map :id (:data response)))]@@ -1640,7 +1639,7 @@:model/Document _ {:name "F Document sorttest"}:model/Segment _ {:name "G Segment sorttest"}:model/Measure _ {:name "H Measure sorttest"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1705,7 +1704,7 @@:table_id table1-id}:model/Measure _ {:name "Measure with Table 2 sorttest":table_id table2-id}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1735,7 +1734,7 @@:collection_id collection1-id}:model/NativeQuerySnippet _ {:name "SQL snippets sorttest"}:model/Transform _ {:name "Transforms sorttest"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1765,7 +1764,7 @@(break-model-card! model-card));; Run analysis in a fresh metadata provider cache session to detect the broken reference(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card)))(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=brokentest")](is (= [(:id model-card)] (mapv :id (:data response)))@@ -1790,7 +1789,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "filtering by card returns only card sources"@@ -1825,7 +1824,7 @@(break-model-card! (t2/select-one :model/Card :id (:id archived-model))));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=archivedbrokentestcard")@@ -1860,7 +1859,7 @@(run-analysis-for-card! (:id card)))(testing "the dependent card's analysis is now broken, attributing the inactive table as the source"(is (false? (t2/select-one-fn :result :model/AnalysisFinding- :analyzed_entity_type "card" :analyzed_entity_id (:id card))))+ :entity_type "card" :entity_id (:id card))))(is (t2/exists? :model/AnalysisFindingError:source_entity_type "table" :source_entity_id (:id table):analyzed_entity_type "card" :analyzed_entity_id (:id card))))@@ -1886,7 +1885,7 @@(break-model-card! model-card));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=multipledependents")@@ -1915,7 +1914,7 @@(break-model-card! model-regular));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "include-personal-collections=false (default) excludes source cards in personal collections"@@ -1948,7 +1947,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(is (=? {:data [{:id (:id model-card-1)}]@@ -1981,7 +1980,7 @@(break-model-card! model-card-b));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-a))(run-analysis-for-card! (:id dependent-card-b)))(doseq [sort-direction [:asc :desc]]@@ -2018,7 +2017,7 @@(break-model-card! model-in-coll2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(doseq [sort-direction [:asc :desc]]@@ -2055,7 +2054,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1a))(run-analysis-for-card! (:id dependent-card-1b))(run-analysis-for-card! (:id dependent-card-2a)))@@ -2094,7 +2093,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1a))(run-analysis-for-card! (:id dependent-card-1b))(run-analysis-for-card! (:id dependent-card-2a)))@@ -2139,7 +2138,7 @@(create-dependent! base-card user "Alpha")(create-dependent! base-card user "Beta")(create-dependent! base-card user "Gamma")- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "filters by name"(is (=? [{:data {:name "Alpha"}}](get-dependents base-card-id :query "Alpha"))))@@ -2155,7 +2154,7 @@(mt/with-temp [:model/Collection {coll-id :id} {:name "SpecialCollection"}](create-dependent! base-card user "Card in root")(create-dependent! base-card user "Card in collection" :collection_id coll-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:data {:name "Card in collection"}}](get-dependents base-card-id :query "SpecialCollection")))))))@@ -2167,7 +2166,7 @@(create-dependent! base-card user "In Personal" :collection_id (personal-collection-id user-id))(create-dependent! base-card user "In Sub" :collection_id sub-coll-id)(create-dependent! base-card user "Regular")- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "default excludes personal collections"(is (=? [{:data {:name "Regular"}}](get-dependents base-card-id))))@@ -2180,7 +2179,7 @@(create-dependent! base-card user "C Card")(create-dependent! base-card user "A Card")(create-dependent! base-card user "B Card")- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:data {:name "A Card"}} {:data {:name "B Card"}} {:data {:name "C Card"}}](get-dependents base-card-id :sort-column :name :sort-direction :asc)))(is (=? [{:data {:name "C Card"}} {:data {:name "B Card"}} {:data {:name "A Card"}}]@@ -2195,7 +2194,7 @@(create-dependent! base-card user "In C" :collection_id coll-c)(create-dependent! base-card user "In A" :collection_id coll-a)(create-dependent! base-card user "In B" :collection_id coll-b)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:data {:name "In A"}} {:data {:name "In B"}} {:data {:name "In C"}}](get-dependents base-card-id :sort-column :location :sort-direction :asc)))(is (=? [{:data {:name "In C"}} {:data {:name "In B"}} {:data {:name "In A"}}]@@ -2207,7 +2206,7 @@(let [{low-id :id} (create-dependent! base-card user "Low"){mid-id :id} (create-dependent! base-card user "Mid"){high-id :id} (create-dependent! base-card user "High")]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(t2/update! :model/Card low-id {:view_count 10})(t2/update! :model/Card mid-id {:view_count 50})(t2/update! :model/Card high-id {:view_count 100})@@ -2226,7 +2225,7 @@(t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id base-card-id:row 0 :col 0 :size_x 4 :size_y 4})(events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(t2/update! :model/Card low-id {:view_count 10})(t2/update! :model/Card high-id {:view_count 100})(is (=? [{:data {:view_count 10}} {:data {:view_count 100}} {:data {:view_count 200}}]@@ -2241,7 +2240,7 @@(create-dependent! base-card user "A Match")(create-dependent! base-card user "B Match")(create-dependent! base-card user "Should not appear")- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}](get-dependents base-card-id :query "Match" :sort-column :name :sort-direction :asc))))))@@ -2270,7 +2269,7 @@:analyzed_entity_id card-id:source_entity_type nil:source_entity_id nil- :error_type "invalid-query"}])+ :error_type "syntax-error"}])(let [result (#'deps.api/node-errors {:card [card-id]})card-errors (get result [:card card-id])](testing "includes errors with visible source"@@ -2278,7 +2277,7 @@(testing "excludes errors with archived source"(is (not (contains? card-errors {:type :missing-column :detail "col2"}))))(testing "includes errors with nil source"- (is (contains? card-errors {:type :invalid-query}))))))))+ (is (contains? card-errors {:type :syntax-error}))))))))(deftest ^:synchronized node-downstream-errors-filtering-test(testing "node-downstream-errors filters by analyzed entity visibility"@@ -2332,7 +2331,7 @@(break-model-card! model-card-b));; Run analysis to detect broken references(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id visible-dep-a))(run-analysis-for-card! (:id visible-dep-b))(run-analysis-for-card! (:id archived-dep-a))@@ -2475,7 +2474,7 @@(mt/with-temp [:model/Card _ {:name "A Card - unreftest" :archived true}:model/Card {card2-id :id} {:name "B Card - unreftest"}:model/Card {card3-id :id} {:name "C Card - unreftest"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "card"@@ -2506,7 +2505,7 @@:archived true}](doseq [id [card1-id (:id card2) card3-id card4-id card5-id]](events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id id) :user-id (mt/user->id :crowberto)}))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "card"@@ -2545,7 +2544,7 @@:type :query:query {:source-table table-1-id}}}](events/publish-event! :event/card-create {:object card :user-id (:creator_id card)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "after creating card that depends on table-1, only table-2 is unused"(is (= #{table-2-id}(->> (mt/user-http-request :crowberto :get 200 "table" :unused-only true)@@ -2575,7 +2574,7 @@(break-model-card! model-card-1)(break-model-card! model-card-2))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "returns only broken dependents for the specified source (not other sources)"@@ -2602,7 +2601,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card-a))(run-analysis-for-card! (:id dependent-card-b)))(testing "count matches dependents-with-errors from /graph/breaking"@@ -2656,7 +2655,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card)));; Admin can access(is (sequential? (mt/user-http-request :crowberto :get 200@@ -2691,7 +2690,7 @@(break-model-card! model-card-1));; Run analysis for both model-2 and dependent-card(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id model-card-2))(run-analysis-for-card! (:id dependent-card)));; Query /graph/broken for model-1 - should only return model-2, not dependent-card@@ -2715,7 +2714,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-card)))(testing "filtering by card type returns the dependent card"(let [response (mt/user-http-request :crowberto :get 200@@ -2748,7 +2747,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(run-analysis-for-card! (:id dependent-in-personal))(run-analysis-for-card! (:id dependent-regular)))(testing "include-personal-collections=false (default) excludes broken dependents in personal collections"@@ -2773,7 +2772,7 @@:owner_user_id (mt/user->id :crowberto)}:model/Table {table2-id :id} {:name "Email Owned Table - ownertest":owner_email "[email protected]"}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "table" :query "ownertest"@@ -2807,7 +2806,7 @@:query (lib/query mp products)}:target {:schema "PUBLIC":name "email_owned_transform_table"}}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "transform" :query "ownertest"@@ -2866,7 +2865,7 @@:database_id (mt/id):dataset_query (delete-venue-query)}](mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [dependent-ids (fn [](->> (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id (mt/id :venues)@@ -2896,7 +2895,7 @@:database_id (mt/id):dataset_query (delete-venue-query)}](mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [unreferenced (fn [query](mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced":types "action"@@ -2910,3 +2909,11 @@(testing "a search for the data-actions root name finds the action"(is (matches? "Data actions"))(is (not (matches? "Our analytics"))))))))))++(deftest backfill-status-requires-superuser-test+ (testing "GET /api/ee/dependencies/backfill-status scans every entity, so only superusers can call it"+ (mt/with-premium-features #{:dependencies}+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 "ee/dependencies/backfill-status")))+ (is (=? {:complete boolean?}+ (mt/user-http-request :crowberto :get 200 "ee/dependencies/backfill-status"))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.cljdeleted file mode 100644index 9dee6dc38fb..00000000000--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj+++ /dev/null@@ -1,401 +0,0 @@-(ns metabase-enterprise.dependencies.findings-test- (:require- [clojure.test :refer :all]- [medley.core :as m]- [metabase-enterprise.dependencies.analysis :as deps.analysis]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]- [metabase-enterprise.dependencies.models.analysis-finding-error :as models.analysis-finding-error]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.test-metadata :as meta]- [metabase.lib.test-util :as lib.tu]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [toucan2.core :as t2]))--(use-fixtures :once (fixtures/initialize :db))--(defn- backfill-all-entity-analyses! []- (doseq [model [:card :transform :segment]]- (while (pos? (deps.findings/analyze-batch! model 100)))))--(deftest ^:synchronized can-analyze-entity-batches-test- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Card {other-card-id :id} {:dataset_query (lib/query mp orders)}]- (is (= 2 (deps.findings/analyze-batch! :card 2)))- (is (= #{models.analysis-finding/*current-analysis-finding-version*}- (t2/select-fn-set :analysis_version- :model/AnalysisFinding- :analyzed_entity_id [:in [card-id other-card-id]]- :analyzed_entity_type :card)))))))--(deftest ^:synchronized does-not-repeatedly-analyze-entities-test- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}- :model/Card _ {:dataset_query (lib/query mp orders)}]- (is (= 2 (deps.findings/analyze-batch! :card 2)))- (is (= 0 (deps.findings/analyze-batch! :card 2)))))))--(deftest ^:synchronized re-analyze-entities-when-analysis-version-bumped-test- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))- orders (lib.metadata/table mp (mt/id :orders))]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}- :model/Card _ {:dataset_query (lib/query mp orders)}]- (is (= 2 (deps.findings/analyze-batch! :card 2)))- (is (= 0 (deps.findings/analyze-batch! :card 2)))- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]- (is (= 2 (deps.findings/analyze-batch! :card 2))))))))--(deftest ^:synchronized does-analyze-native-entities-test- (testing "failed analysis result is appropriately stored in the appdb"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query mp "utter nonsense")}]- (is (= 1 (deps.findings/analyze-batch! :card 1)))- (is (= [models.analysis-finding/*current-analysis-finding-version* false]- (t2/select-one-fn (juxt :analysis_version :result)- :model/AnalysisFinding- :analyzed_entity_id card-id- :analyzed_entity_type :card))))))))--(deftest ^:synchronized does-report-errors-for-missing-refs-in-fields-test- (testing "missing (not inactive) field refs in :fields are reported as findings (GHY-3157)"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- orders (lib.metadata/table mp (mt/id :orders))- base (lib/query mp orders)- cols (vec (take 5 (lib/returned-columns base orders)))- bad-col (-> (first cols)- lib/ref- (assoc 2 "bad_column"))- query (lib/with-fields base (conj cols bad-col))]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]- (is (= 1 (deps.findings/analyze-batch! :card 1)))- (is (= [models.analysis-finding/*current-analysis-finding-version* false]- (t2/select-one-fn (juxt :analysis_version :result)- :model/AnalysisFinding- :analyzed_entity_id card-id- :analyzed_entity_type :card))))))))--;; TODO: (bshepherdson, 2026-02-05) Add a test like does-not-report-errors-in-removable-refs-test-1-stage-fields for-;; join clause :fields as well. See QUE-3081 and QUE-3044.--(deftest ^:parallel reports-missing-field-in-downstream-card-fields-test- (testing "Q2 based on Q1 with :fields referencing a column Q1 no longer returns should be reported (GHY-3157)"- (let [mp meta/metadata-provider- q1 (lib/query mp (meta/table-metadata :orders))- q1-cols (lib/returned-columns q1)- removed-col (m/find-first #(= (:name %) "ID") q1-cols)- ;; Card 101 = Q1 with ID removed from result-metadata (simulates Q1 dropping that field)- mp (lib.tu/metadata-provider-with-card-from-query- mp 101 q1- {:result-metadata (vec (remove #(= (:name %) "ID") q1-cols))})- ;; Q2: query based on card 101, with :fields including the now-missing ID- q2-base (lib/query mp {:lib/type :metadata/card :id 101})- q2-cols (lib/returned-columns q2-base)- q2 (lib/with-fields q2-base (conj (vec q2-cols) (lib/ref removed-col)))- ;; Card 102 = Q2 (provide result-metadata explicitly since q2 has a bad ref)- mp (lib.tu/metadata-provider-with-card-from-query- mp 102 q2- {:result-metadata (vec q2-cols)})- results (deps.analysis/check-entity mp :card 102)]- (is (seq results)- "Q2 should have findings for the missing ID column from Q1"))))--(defn- stale-map- "Returns a map of {entity-id stale?} for the given card IDs."- [card-ids]- (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id [:in card-ids]))--(defn- finding-stale?- "Returns the stale value for a specific entity's analysis finding, or nil if no finding exists."- [entity-type entity-id]- (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type entity-type- :analyzed_entity_id entity-id))--(deftest ^:synchronized analyze-batch-picks-up-missing-analyses-test- (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]- (testing "card has no analysis finding initially"- (is (not (t2/exists? :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))- (testing "analyze-batch! creates analysis for the card"- (lib-be/with-metadata-provider-cache- (is (pos? (deps.findings/analyze-batch! :card 10)))))- (testing "card now has an analysis finding"- (is (t2/exists? :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)))))))))--(deftest mark-dependents-stale-test- (testing "mark-transitive-dependents-stale! marks direct dependents as stale"- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}- :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-card-id- :to_entity_type :card- :to_entity_id parent-card-id}]- (run! deps.findings/upsert-analysis!- (t2/select :model/Card :id [:in [parent-card-id child-card-id]]))- (is (= {parent-card-id false, child-card-id false}- (stale-map [parent-card-id child-card-id]))- "neither should be stale before marking")- (t2/with-transaction [_conn]- (deps.findings/mark-transitive-dependents-stale! {:card [parent-card-id]})- (is (= {parent-card-id false, child-card-id true}- (stale-map [parent-card-id child-card-id]))- "parent should NOT be stale, child should be stale")))))))))--(deftest mark-dependents-stale-transitive-test- (testing "mark-transitive-dependents-stale! marks transitive dependents as stale"- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {grandparent-id :id} {:dataset_query (lib/query mp products)}- :model/Card {parent-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp grandparent-id))}- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id parent-id- :to_entity_type :card- :to_entity_id grandparent-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}]- (run! deps.findings/upsert-analysis!- (t2/select :model/Card :id [:in [grandparent-id parent-id child-id]]))- (t2/with-transaction [_conn]- (deps.findings/mark-transitive-dependents-stale! {:card [grandparent-id]})- (is (= {grandparent-id false, parent-id true, child-id true}- (stale-map [grandparent-id parent-id child-id]))- "grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))--(deftest ^:synchronized mark-subtree-stale-then-drain-test- (testing "marking an entity stale also marks its whole transitive subtree up front, and check-entities! drains it"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {gp-id :id :as gp-card} {:dataset_query (lib/query mp products)}- :model/Card {p-id :id :as p-card} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}- :model/Card {c-id :id :as c-card} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}- :model/Dependency _ {:from_entity_type :card :from_entity_id p-id- :to_entity_type :card :to_entity_id gp-id}- :model/Dependency _ {:from_entity_type :card :from_entity_id c-id- :to_entity_type :card :to_entity_id p-id}]- ;; Analyze all three so they have findings- (run! deps.findings/upsert-analysis! [gp-card p-card c-card])- (is (= {gp-id false, p-id false, c-id false}- (stale-map [gp-id p-id c-id]))- "all should start as not stale")- ;; A change to the grandparent marks it AND its whole transitive subtree (parent and child) stale up front- (deps.findings/mark-entity-and-transitive-dependents-stale! :card gp-id)- (is (= {gp-id true, p-id true, c-id true}- (stale-map [gp-id p-id c-id]))- "the grandparent and its whole subtree should be stale")- ;; The drain analyzes each once, without re-marking anything stale during draining- (#'task.entity-check/check-entities!)- (is (= {gp-id false, p-id false, c-id false}- (stale-map [gp-id p-id c-id]))- "after check-entities!, the whole subtree should be re-analyzed (not stale)"))))))))--(deftest ^:synchronized transitive-marking-through-non-analyzable-entity-test- (testing "marking dependents stale reaches analyzable entities beyond non-analyzable intermediaries (e.g., tables)"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))- products-id (mt/id :products)]- (mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (mt/with-model-cleanup [:model/AnalysisFinding]- ;; Chain: card → (depends on) table → (depends on) transform.- ;; The table is not analyzable, but the card beyond it should still be reached.- (mt/with-temp [:model/Transform {transform-id :id :as transform}- {:source {:type :query :query (lib/query mp products)}- :name "test_transitive_transform"- :target {:schema "public" :name "transitive_test_table" :type :table}}- :model/Card {card-id :id :as card}- {:dataset_query (lib/query mp products)}- ;; table depends on transform (downstream of transform)- :model/Dependency _ {:from_entity_type :table :from_entity_id products-id- :to_entity_type :transform :to_entity_id transform-id}- ;; card depends on the same table- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id products-id}]- ;; Analyze the transform and card (tables aren't analyzable)- (deps.findings/upsert-analysis! transform)- (deps.findings/upsert-analysis! card)- (is (false? (finding-stale? :card card-id))- "card should start as not stale")- ;; The card is two hops downstream of the transform: transform → table → card- (deps.findings/mark-transitive-dependents-stale! {:transform [transform-id]})- (is (true? (finding-stale? :card card-id))- "card should be stale — transitive marking reached it via the non-analyzable table")- ;; Run entity-check to drain stale entities- (#'task.entity-check/check-entities!)- (is (false? (finding-stale? :card card-id))- "card should be re-analyzed after entity-check"))))))))--(deftest ^:synchronized cyclic-dependency-drain-terminates-test- (testing "check-entities! terminates on a cyclic dependency graph — staleness is fixed up front and never re-marked- during the drain"- (backfill-all-entity-analyses!)- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {a-id :id :as a-card} {:dataset_query (lib/query mp products)}- :model/Card {b-id :id :as b-card} {:dataset_query (lib/query mp products)}- ;; cycle: a depends on b, and b depends on a- :model/Dependency _ {:from_entity_type :card :from_entity_id a-id- :to_entity_type :card :to_entity_id b-id}- :model/Dependency _ {:from_entity_type :card :from_entity_id b-id- :to_entity_type :card :to_entity_id a-id}]- (run! deps.findings/upsert-analysis! [a-card b-card])- (deps.findings/mark-entity-and-transitive-dependents-stale! :card a-id)- (is (= {a-id true, b-id true}- (stale-map [a-id b-id]))- "both cycle members should be marked stale")- ;; Under the old wave-propagation design this drain would loop forever; bound it so a regression fails fast.- (let [done (future (#'task.entity-check/check-entities!))]- (is (not= ::timeout (deref done 30000 ::timeout))- "check-entities! must terminate on a cyclic graph")- (is (= {a-id false, b-id false}- (stale-map [a-id b-id]))- "both cycle members should be drained (re-analyzed, not stale)")))))))))--(deftest ^:synchronized python-transform-records-clean-finding-test- (testing "GHY-3584: a Python transform has no query, so upsert-analysis! resolves its database from source_database_id and records a clean finding"- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :python- :source-database (mt/id)- :source-tables []- :body "def transform():\n pass"}}]- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! (t2/select-one :model/Transform transform-id)))- (is (=? {:result true :stale false}- (t2/select-one :model/AnalysisFinding :analyzed_entity_type :transform :analyzed_entity_id transform-id)))- (is (empty? (models.analysis-finding-error/errors-for-entity :transform transform-id))))))))--(deftest ^:synchronized unanalyzable-entity-records-terminal-error-test- (testing "A stale entity whose database can't be resolved records a terminal error finding instead of no-oping forever."- ;; Without the fix, instance-db-id -> nil makes upsert-analysis! no-op: the stale flag is never- ;; cleared, so the entity is re-selected and re-attempted on every run. The fix records a terminal- ;; error, clearing the flag.- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]- (mt/with-temp [:model/Card card {:dataset_query (mt/native-query {:query "SELECT 1"})- :database_id (mt/id)}]- (let [cid (:id card)]- ;; seed a clean finding, mark it stale, then make its database unresolvable- (models.analysis-finding/upsert-analysis! :card cid true [])- (models.analysis-finding/mark-stale! {:card #{cid}})- (with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}- (fn []- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))))- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id cid))- "the stale flag is cleared, so the entity stops being re-selected forever")- (is (seq (models.analysis-finding-error/errors-for-entity :card cid))- "an error finding is recorded explaining why the entity couldn't be analyzed")))))))--(deftest ^:synchronized unanalyzable-entity-rechecks-when-database-resolvable-again-test- (testing "After a no-DB terminal error, the entity is analyzed normally once its database resolves again."- ;; The terminal error finding is not sticky: it clears the stale flag but does not block future- ;; analysis. When the entity is next marked stale and its database now resolves, upsert-analysis!- ;; takes the normal path and replaces the terminal error with a real analysis result.- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]- (let [mp (mt/metadata-provider)- products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Card card {:dataset_query (lib/query mp products)}]- (let [cid (:id card)]- ;; phase 1: database unresolvable -> terminal error recorded, stale cleared- (with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}- (fn []- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))))- (is (seq (models.analysis-finding-error/errors-for-entity :card cid))- "phase 1: a terminal error is recorded while the database is unresolvable")- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id cid))- "phase 1: stale flag cleared")- ;; phase 2: database resolvable again; re-trigger analysis (as an event/job would)- (models.analysis-finding/mark-stale! {:card #{cid}})- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! (t2/instance :model/Card card)))- (is (empty? (models.analysis-finding-error/errors-for-entity :card cid))- "phase 2: re-analyzed normally — the terminal db error is replaced by a clean result")- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id cid))- "phase 2: re-analysis cleared stale again"))))))))--(deftest ^:synchronized analyze-batch-shares-one-metadata-provider-test- (testing "Every entity in an analysis batch must share one MetadataProvider per database. Reading an entity attaches- a provider to its query, so selecting the batch outside the cache gives each entity a private one. Nothing- reads through those providers today -- upsert-analysis! builds its own from the database id and passes it- explicitly -- so this is a latent trap rather than a live leak: the moment an entity is analyzed through its- own query, a batch of 500 retains 500 providers' worth of fetched metadata. upsert-analysis! already throws- when no cache is bound, so the cache is meant to cover this work; only the select sat outside it."- (backfill-all-entity-analyses!)- (let [providers (atom [])- native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}- :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]- (let [wanted #{card1-id card2-id}- upsert-analysis! (mt/original-fn #'deps.findings/upsert-analysis!)]- (mt/with-dynamic-fn-redefs [deps.findings/upsert-analysis!- (fn [instance]- (when (wanted (:id instance))- (swap! providers conj (:lib/metadata (:dataset_query instance))))- (upsert-analysis! instance))]- (deps.findings/analyze-batch! :card 500)))- (testing "both cards were selected, so the assertion below is meaningful"- (is (= 2 (count @providers))))- (testing "and they carry the same provider instance"- ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be- ;; satisfied by two separate caches and would not detect the regression.- (is (every? #(identical? (first @providers) %) @providers))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.cljindex 15586d21787..c731c2d78fb 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj@@ -1,16 +1,15 @@(ns metabase-enterprise.dependencies.hooks-test(:require[clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.hooks]- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.task.backfill :as task.backfill][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.serialization.test-util :as ts][metabase-enterprise.serialization.v2.load :as serdes.load][metabase.batch-processing.core :as grouper]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata][metabase.models.serialization :as serdes][metabase.test :as mt][toucan2.core :as t2]))@@ -18,173 +17,357 @@(commentmetabase-enterprise.dependencies.hooks/keep-me)-(defn- stale? [entity-type entity-id]- (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true))+(def ^:private table-query deps.test/table-query)-(defn- table-query [table]- (let [mp (mt/metadata-provider)]- (lib/query mp (lib.metadata/table mp (mt/id table)))))+(def ^:private card-query deps.test/card-query)-(defn- card-query [card-id]- (let [mp (mt/metadata-provider)]- (lib/query mp (lib.metadata/card mp card-id))))+(def ^:private pending? deps.test/dependency-pending?)-(deftest card-update-test- (testing "updating a card marks it stale only when a column its dependencies come from changes"- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/DependencyStatus]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]- (deps.dependency-status/upsert-status! :card card-id false)- (t2/update! :model/Card card-id {:name "Renamed"})- (is (not (stale? :card card-id)))- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})- (is (stale? :card card-id)))))))--(deftest card-update-marks-finding-test- (testing "updating a card's query marks its own analysis finding stale in the same transaction"+(def ^:private computed! deps.test/record-dependencies-computed!)++(defn- status-row-count []+ (reduce + (map t2/count [:model/DependencyStatus :model/ResultMetadataStatus :model/AnalysisFindingStatus])))++(deftest card-update-adds-no-status-test+ (testing "updating a card adds no status row, and triggers the jobs once it commits"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/DependencyStatus :model/AnalysisFinding]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]- (deps.analysis-finding/upsert-analysis! :card card-id true [])- (let [finding-stale? #(t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id)]- (t2/update! :model/Card card-id {:name "Renamed"})- (is (not (finding-stale?)))- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})- (is (finding-stale?))))))))+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (let [submitted (atom [])+ before (status-row-count)]+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (deps.test/do-with-immediate-hook-writes+ #(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})))+ (is (= before (status-row-count)))+ (is (= [{:dependency-jobs true}] @submitted)))))))(deftest card-update-without-feature-test- (testing "without the dependencies feature the hooks do nothing"+ (testing "without the dependencies feature the hooks trigger nothing"(mt/with-premium-features #{}- (mt/with-model-cleanup [:model/DependencyStatus]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]- (deps.dependency-status/upsert-status! :card card-id false)- (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})- (is (not (stale? :card card-id))))))))--(deftest dashboard-card-test- (testing "inserting, updating and deleting a dashboard card, or adding a series, marks its dashboard stale"+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (let [submitted (atom [])]+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (deps.test/do-with-immediate-hook-writes+ #(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})))+ (is (= [] @submitted)))))))++(deftest update-marking-nothing-triggers-nothing-test+ (testing "an update that marks nothing triggers no job, as a sync storing only a field's fingerprint"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Field {field-id :id} {:table_id (mt/id :venues) :name "x"}]+ (let [submitted (atom [])]+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (deps.test/do-with-immediate-hook-writes+ #(t2/update! :model/Field field-id {:last_analyzed (t/offset-date-time)})))+ (is (= [] @submitted)))))))++(deftest output-columns-mark-dependents-test+ (testing "a metric rename, a field position and a table schema reach the cards reading them, a question rename and a+ table caveat don't"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}+ :model/Card {metric-id :id} {:type :metric+ :dataset_query (lib/aggregate (table-query :venues) (lib/count))}]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :card :to_entity_id source-id})+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :card :to_entity_id metric-id})+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id source-id+ :to_entity_type :table :to_entity_id (mt/id :venues)})+ (let [marks? (fn [card write!]+ (deps.test/record-result-metadata-processed! card)+ (write!)+ (deps.test/result-metadata-pending? card))]+ (is (not (marks? card-id #(t2/update! :model/Card source-id {:name "Renamed"}))))+ (is (marks? card-id #(t2/update! :model/Card metric-id {:name "Renamed"})))+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:position 5}+ (is (marks? source-id #(t2/update! :model/Field (mt/id :venues :price) {:position 0}))))+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:schema "PUBLIC"}+ (is (marks? source-id #(t2/update! :model/Table (mt/id :venues) {:schema "OTHER"}))))+ (mt/with-temp [:model/TableUserSettings _ {:table_id (mt/id :venues)}]+ (is (not (marks? source-id #(t2/update! :model/TableUserSettings (mt/id :venues)+ {:caveats "Careful"}))))))))))++(deftest card-metadata-dependents-see-test+ (testing "a card's result metadata change marks its dependents only when it changes what they see"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :card :to_entity_id source-id})+ (let [metadata (t2/select-one-fn :result_metadata :model/Card :id source-id)+ marks? (fn [f]+ (deps.test/record-result-metadata-processed! card-id)+ (t2/update! :model/Card source-id {:result_metadata (mapv f metadata)})+ (deps.test/result-metadata-pending? card-id))]+ (is (not (marks? #(assoc % :fingerprint {:global {:distinct-count 3}}))))+ (is (marks? #(cond-> % (= "NAME" (:name %)) (assoc :display_name "Renamed")))))))))++(defn- with-card-and-dependent!+ "Call `f` with a card on `table-key` and a card on that card, whose result metadata and analysis are current, and a+ function telling whether both are pending for both jobs."+ [table-key f]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query table-key)}+ :model/Card {dependent-id :id} {:dataset_query (card-query card-id)}]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id table-key)})+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id dependent-id+ :to_entity_type :card :to_entity_id card-id})+ (f (fn []+ (doseq [id [card-id dependent-id]]+ (deps.test/record-result-metadata-processed! id)+ (deps.test/record-analysis-finding-processed! :card id)))+ (fn []+ (every? (fn [id] (and (deps.test/result-metadata-pending? id)+ (deps.test/analysis-finding-pending? :card id)))+ [card-id dependent-id])))))++(deftest remap-marks-transitive-dependents-test+ (testing "an external remap, its human-readable field and that field's user settings mark every card that carries+ the remapped column"+ (mt/with-premium-features #{:dependencies}+ (with-card-and-dependent!+ :orders+ (fn [processed! pending?]+ (processed!)+ (mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :product_id)+ :type :external+ :name "Product"+ :human_readable_field_id (mt/id :products :title)}]+ (testing "adding the remap"+ (is (pending?)))+ (testing "changing its human-readable field"+ (processed!)+ (mt/with-temp-vals-in-db :model/Field (mt/id :products :title) {:display_name "Renamed"}+ (is (pending?))))+ (testing "a user setting of its human-readable field"+ (processed!)+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :title) :display_name "Item"}]+ (is (pending?))))))))))++(deftest internal-remap-values-mark-transitive-dependents-test+ (testing "the FieldValues of an internally remapped field mark every card carrying the remapped column when they gain+ or lose human-readable values, and not when only their values change"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/DependencyStatus]+ (with-card-and-dependent!+ :venues+ (fn [processed! pending?]+ (mt/with-temp [:model/Dimension _ {:field_id (mt/id :venues :price) :type :internal :name "Price"}+ :model/FieldValues fv {:field_id (mt/id :venues :price) :type :full+ :values [1 2 3 4] :human_readable_values []}]+ (processed!)+ (t2/update! :model/FieldValues (:id fv) {:values [1 2 3 4 5]})+ (is (not (pending?)))+ (t2/update! :model/FieldValues (:id fv) {:human_readable_values ["$" "$$" "$$$" "$$$$" "$$$$$"]})+ (is (pending?))))))))++(deftest batched-marks-test+ (testing "a batch of field writes marks the cards reading their table once per status, and triggers the jobs once"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :venues)})+ (deps.test/record-result-metadata-processed! card-id)+ (let [marks (atom [])+ submitted (atom [])+ mark! (mt/original-fn #'dependencies.db/mark-dependents-result-metadata-statuses-stale!)]+ (mt/with-dynamic-fn-redefs [dependencies.db/mark-dependents-result-metadata-statuses-stale!+ (fn [entity-type entity-ids]+ (swap! marks conj [entity-type entity-ids])+ (mark! entity-type entity-ids))+ grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:position 5}+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :name) {:position 6}+ (reset! marks [])+ (reset! submitted [])+ (deps.test/do-with-immediate-hook-writes+ #(dependencies.hooks/with-batched-marks+ (t2/update! :model/Field (mt/id :venues :price) {:position 0})+ (t2/update! :model/Field (mt/id :venues :name) {:position 1})))+ (is (= [[:table #{(mt/id :venues)}]] @marks))+ (is (= [{:dependency-jobs true}] @submitted))+ (is (deps.test/result-metadata-pending? card-id))))))))))++(deftest snippet-delete-marks-transitive-dependencies-test+ (testing "deleting a snippet nested in another marks the dependencies of the cards expanding it"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner delete" :content "VENUES"}+ :model/NativeQuerySnippet {outer-id :id} {:name "outer delete" :content "{{snippet: inner delete}}"}+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "{{snippet: outer delete}}")}]+ (t2/insert! :model/Dependency {:from_entity_type :snippet :from_entity_id outer-id+ :to_entity_type :snippet :to_entity_id inner-id})+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :snippet :to_entity_id outer-id})+ (computed! :card card-id)+ (t2/delete! :model/NativeQuerySnippet :id inner-id)+ (is (pending? :card card-id))))))++(deftest card-marks-own-statuses-test+ (testing "an update of what a job reads of a card marks the card's own status for that job stale, and a question's+ rename marks nothing"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (computed! :card card-id)+ (t2/update! :model/Card card-id {:name "Renamed"})+ (is (not (pending? :card card-id)))+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})+ (is (pending? :card card-id))))))++(deftest dashboard-parts-mark-dashboard-test+ (testing "inserting, updating and deleting a dashboard card, a series or a tab marks the dashboard's dependencies stale"+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}(mt/with-temp [:model/Dashboard {dashboard-id :id} {}:model/Card {card-id :id} {:dataset_query (table-query :products)}:model/Card {other-id :id} {:dataset_query (table-query :orders)}]- (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id false)]- (processed!)- (let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard- {:dashboard_id dashboard-id :card_id card-id- :row 0 :col 0 :size_x 4 :size_y 4})]- (testing "inserting a dashboard card"- (is (stale? :dashboard dashboard-id)))- (processed!)- (t2/update! :model/DashboardCard dashcard-id {:row 1})- (testing "moving a dashboard card"- (is (not (stale? :dashboard dashboard-id))))- (t2/update! :model/DashboardCard dashcard-id {:card_id other-id})- (testing "pointing a dashboard card at another card"- (is (stale? :dashboard dashboard-id)))- (processed!)- (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dashcard-id :card_id card-id :position 0})- (testing "adding a series"- (is (stale? :dashboard dashboard-id)))- (processed!)- (t2/delete! :model/DashboardCard :id dashcard-id)- (testing "removing a dashboard card"- (is (stale? :dashboard dashboard-id))))))))))+ (let [changes? (fn [write!]+ (computed! :dashboard dashboard-id)+ (write!)+ (pending? :dashboard dashboard-id))+ dashcard (atom nil)]+ (testing "inserting a dashboard card"+ (is (changes? #(reset! dashcard (t2/insert-returning-pk! :model/DashboardCard+ {:dashboard_id dashboard-id :card_id card-id+ :row 0 :col 0 :size_x 4 :size_y 4})))))+ (testing "pointing a dashboard card at another card"+ (is (changes? #(t2/update! :model/DashboardCard @dashcard {:card_id other-id}))))+ (testing "adding a series"+ (is (changes? #(t2/insert! :model/DashboardCardSeries {:dashboardcard_id @dashcard :card_id card-id+ :position 0}))))+ (testing "removing a series"+ (is (changes? #(t2/delete! :model/DashboardCardSeries :dashboardcard_id @dashcard))))+ (testing "adding and removing a tab"+ (let [tab-id (atom nil)]+ (is (changes? #(reset! tab-id (t2/insert-returning-pk! :model/DashboardTab+ {:dashboard_id dashboard-id :name "Tab"+ :position 0}))))+ (is (changes? #(t2/delete! :model/DashboardTab :id @tab-id)))))+ (testing "removing a dashboard card"+ (is (changes? #(t2/delete! :model/DashboardCard :id @dashcard))))+ (testing "nothing else"+ (computed! :dashboard dashboard-id)+ (is (not (pending? :dashboard dashboard-id))))))))))++(deftest table-parts-mark-cards-test+ (testing "a write of a table, its user settings, one of its fields, field user settings or dimensions, marks the cards+ on it stale"+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :venues)})+ (let [table-id (mt/id :venues)+ field-id (mt/id :venues :price)+ changes? (fn [write!]+ (deps.test/record-result-metadata-processed! card-id)+ (write!)+ (deps.test/result-metadata-pending? card-id))]+ (mt/with-temp-vals-in-db :model/Table table-id {:display_name "Venues"}+ (is (changes? #(t2/update! :model/Table table-id {:display_name "Places"}))))+ (mt/with-temp-vals-in-db :model/Field field-id {:display_name "Price"}+ (is (changes? #(t2/update! :model/Field field-id {:display_name "Cost"}))))+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id :display_name "Cost"}]+ (is (changes? (fn [] (t2/update! :model/FieldUserSettings :field_id field-id {:display_name "Price"})))))+ (is (changes? #(t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Places"})))+ (is (changes? #(t2/update! :model/TableUserSettings table-id {:display_name "Spots"})))+ (is (changes? #(t2/delete! :model/TableUserSettings :table_id table-id)))+ (let [dimension-id (atom nil)]+ (is (changes? #(reset! dimension-id (t2/insert-returning-pk! :model/Dimension+ {:field_id field-id :name "Price"+ :type :internal}))))+ (is (changes? #(t2/delete! :model/Dimension :id @dimension-id))))))))))(deftest entity-delete-test- (testing "deleting an entity marks its status stale, and the backfill drops its dependencies and status"+ (testing "the dependency job drops the dependencies and status of a deleted entity"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}](t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id:to_entity_type "table" :to_entity_id (mt/id :products)})- (deps.dependency-status/upsert-status! :card card-id false)+ (computed! :card card-id)(t2/delete! :model/Card :id card-id)- (is (stale? :card card-id))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (not (t2/exists? :model/Dependency :from_entity_type "card" :from_entity_id card-id)))(is (not (t2/exists? :model/DependencyStatus :entity_type "card" :entity_id card-id)))))))(deftest concurrent-saves-of-dependent-cards-commit-test(testing "saving a card and then its dependent in one transaction, while another request saves the dependent, commits both"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/DependencyStatus]- (let [insert-card! (fn [query]- (t2/insert-returning-pk! :model/Card {:name (mt/random-name)- :display :table- :visualization_settings {}- :creator_id (mt/user->id :crowberto)- :database_id (mt/id)- :dataset_query query}))- card-a (insert-card! (table-query :products))- card-b (insert-card! (card-query card-a))- a-saved (promise)- b-status-marked (promise)- save! (fn [thunk]- (future- (try- (thunk)- :committed- (catch Throwable e- (ex-message (or (ex-cause e) e))))))- mark-stale! (mt/original-fn #'deps.dependency-status/mark-stale!)]+ (mt/with-model-cleanup [:model/Card]+ (let [insert-card! (fn [query]+ (t2/insert-returning-pk! :model/Card {:name (mt/random-name)+ :display :table+ :visualization_settings {}+ :creator_id (mt/user->id :crowberto)+ :database_id (mt/id)+ :dataset_query query}))+ card-a (insert-card! (table-query :products))+ card-b (insert-card! (card-query card-a))+ a-saved (promise)+ save! (fn [thunk]+ (future+ (try+ (thunk)+ :committed+ (catch Throwable e+ (ex-message (or (ex-cause e) e))))))](t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b:to_entity_type :card :to_entity_id card-a})- (doseq [card-id [card-a card-b]]- (deps.dependency-status/upsert-status! :card card-id false)- (deps.analysis-finding/upsert-analysis! :card card-id true []))- (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!- (fn [type->ids]- (mark-stale! type->ids)- (when (= {:card #{card-b}} type->ids)- (deliver b-status-marked true)))]- (let [cascade (save! #(t2/with-transaction [_conn]- (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})- (deliver a-saved true)- (deref b-status-marked 10000 nil)- (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))- single (save! #(do (deref a-saved 10000 nil)- (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]- (is (= [:committed :committed]- [(deref cascade 30000 :timeout) (deref single 30000 :timeout)])))))))))--(deftest serdes-load-triggers-jobs-once-test- (testing "a serdes load marks what it writes, nested deletes included, and triggers the jobs once it finishes"- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/DependencyStatus]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}- :model/Dashboard {dashboard-id :id} {}- :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]- (let [exported (ts/extract-one "Dashboard" dashboard-id)- submitted (atom [])- triggered (atom 0)]- (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id- :row 4 :col 0 :size_x 4 :size_y 4})- (deps.dependency-status/upsert-status! :dashboard dashboard-id false)- (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))- task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))]- (deps.test/do-with-immediate-hook-writes- #(serdes/with-cache- (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))- (testing "deleting the dashboard card missing from the export marks its dashboard"- (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))- (is (stale? :dashboard dashboard-id)))- (testing "the load triggers the jobs once, after it finishes"- (is (= [] @submitted))- (is (= 1 @triggered)))))))))--(deftest serdes-reload-of-unchanged-dashboard-test- (testing "reloading an unchanged dashboard export, including empty visualization settings, does not mark it"+ (let [cascade (save! #(t2/with-transaction [_conn]+ (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})+ (deliver a-saved true)+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))+ single (save! #(do (deref a-saved 10000 nil)+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]+ (is (= [:committed :committed]+ [(deref cascade 30000 :timeout) (deref single 30000 :timeout)]))))))))++(deftest serdes-load-triggers-jobs-test+ (testing "a serdes load triggers the jobs through the writes it makes"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/DependencyStatus]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}- :model/Dashboard {dashboard-id :id} {}- :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id- :visualization_settings {}}]- (let [exported (ts/extract-one "Dashboard" dashboard-id)]- (deps.dependency-status/upsert-status! :dashboard dashboard-id false)- (mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])]- (deps.test/do-with-immediate-hook-writes- #(serdes/with-cache- (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))- (is (not (stale? :dashboard dashboard-id)))))))))+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}+ :model/Dashboard {dashboard-id :id} {}+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]+ (let [exported (ts/extract-one "Dashboard" dashboard-id)+ submitted (atom [])]+ (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id+ :row 4 :col 0 :size_x 4 :size_y 4})+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (deps.test/do-with-immediate-hook-writes+ #(serdes/with-cache+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))+ (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))+ (is (seq @submitted)))))))++(deftest ^:synchronized database-delete-marks-source-transforms-stale-test+ (testing "deleting a database marks its source transforms' analysis findings stale so they surface on /dependency-diagnostics/broken (GDGT-2447)"+ (deps.test/run-with-dependencies-setup!+ (fn [_mp]+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Transform {transform-id :id}+ {:source {:type :query+ :query {:database db-id+ :type :native+ :native {:query "SELECT 1"}}}+ :target {:schema "public" :name "out" :type :table}}+ :model/AnalysisFinding _ {:entity_type :transform+ :entity_id transform-id+ :result true+ :analyzed_at :%now}]+ (t2/delete! :model/Database db-id)+ (is (nil? (t2/select-one :model/Database db-id))+ "the Database row is actually gone")+ (is (pending? :transform transform-id))+ (deps.test/synchronously-run-dependency-job!)+ (testing "and the analysis finding job flips :result false so the transform shows on broken-diagnostics"+ (deps.test/run-analysis-finding-job!)+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_type :transform+ :entity_id transform-id)))+ (testing "an analysis_finding_error row points the transform at itself so the diagnostics page (`/graph/breaking`) surfaces it"+ (is (=? {:source_entity_type :transform+ :source_entity_id transform-id+ :error_type :validation-exception-error}+ (t2/select-one :model/AnalysisFindingError+ :analyzed_entity_type :transform+ :analyzed_entity_id transform-id))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.cljindex e223a03d437..919f23bc793 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj@@ -1,76 +1,62 @@(ns metabase-enterprise.dependencies.models.analysis-finding-test(:require[clojure.test :refer :all]+ [metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.test :as mt]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent CyclicBarrier TimeUnit)))(set! *warn-on-reflection* true)-(deftest upsert-analysis-clears-stale-flag-test- (testing "upsert-analysis! sets stale to false"- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {}]- (t2/with-transaction [_conn]- ;; Insert a finding with stale = true- (t2/insert! :model/AnalysisFinding- {:analyzed_entity_type :card- :analyzed_entity_id card-id- :analysis_version (dec models.analysis-finding/*current-analysis-finding-version*)- :analyzed_at (java.time.OffsetDateTime/now)- :result true- :stale true})- ;; Verify it's stale- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)))- ;; Now upsert a new analysis- (models.analysis-finding/upsert-analysis! :card card-id true [])- ;; Verify stale is now false- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))))))+(defn- finding [card-id]+ (t2/select-one :model/AnalysisFinding :entity_type :card :entity_id card-id))-(deftest upsert-analysis-new-finding-has-stale-false-test- (testing "upsert-analysis! creates new findings with stale = false"+(deftest ^:synchronized take-pending-analysis-finding-statuses-test+ (testing "the analysis job takes the entities without a status, with one counting a write or one of an older+ analysis, the last ones last, only once their dependencies are current, and inserts the missing statuses+ counting one write"(mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {}]- (t2/with-transaction [_conn]- ;; Upsert a new analysis (no existing finding)- (models.analysis-finding/upsert-analysis! :card card-id true [])- ;; Verify stale is false- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))))))+ (mt/with-model-cleanup [:model/AnalysisFindingStatus :model/DependencyStatus]+ (mt/with-temp [:model/Card {unanalyzed :id} {}+ :model/Card {current :id} {}+ :model/Card {outdated :id} {}+ :model/Card {changed :id} {}+ :model/Card {blocked :id} {}]+ (let [ids #{unanalyzed current outdated changed blocked}+ version models.analysis-finding/*current-analysis-finding-version*+ selected #(into [] (filter (comp ids :entity_id))+ (dependencies.db/take-pending-analysis-finding-statuses!+ :card 100000 version models.dependency/current-dependency-analysis-version))]+ (doseq [card-id (disj ids blocked)]+ (deps.test/record-dependencies-computed! :card card-id))+ (doseq [card-id [current changed]]+ (deps.test/record-status-processed! :model/AnalysisFindingStatus :card card-id version))+ (deps.test/record-status-processed! :model/AnalysisFindingStatus :card outdated (dec version))+ (dependencies.db/mark-analysis-finding-statuses-stale! {:card #{changed}})+ (let [picked (selected)]+ (is (= #{{:entity_id unanalyzed :stale 1} {:entity_id changed :stale 1}} (set (butlast picked))))+ (is (= {:entity_id outdated :stale 0} (last picked)))+ (is (= 1 (t2/select-one-fn :stale :model/AnalysisFindingStatus :entity_type :card+ :entity_id unanalyzed))))))))))-(deftest ^:synchronized instances-for-analysis-serves-oldest-analyzed-first-test- (testing "Among stale entities, instances-for-analysis returns the oldest-analyzed first."- ;; Fairness tiebreak: the primary sort only ranks stale-before-outdated, so among equally-stale- ;; rows the order was DB-arbitrary. A batch smaller than the stale backlog could then keep- ;; serving the same subset, starving the rest across runs. Ordering oldest-`analyzed_at`-first- ;; makes selection round-robin: whatever was just attempted has the newest stamp and sinks to the- ;; back, so every stale entity is reached before any is revisited.- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding]- (mt/with-temp [:model/Card {c1 :id} {}- :model/Card {c2 :id} {}- :model/Card {c3 :id} {}- :model/Card {c4 :id} {}]- (let [base (java.time.OffsetDateTime/of 2020 1 1 0 0 0 0 java.time.ZoneOffset/UTC)- stale! (fn [card-id minutes]- (t2/insert! :model/AnalysisFinding- {:analyzed_entity_type :card- :analyzed_entity_id card-id- :analysis_version models.analysis-finding/*current-analysis-finding-version*- :analyzed_at (.plusMinutes base minutes)- :result true- :stale true}))]- ;; Insert newest-analyzed first (c4 -> c1) so the unordered scan order is the OPPOSITE of- ;; what the tiebreak must produce: without the fix this returns [c4 c3], with it [c1 c2].- (stale! c4 3)- (stale! c3 2)- (stale! c2 1)- (stale! c1 0)- (let [batch (models.analysis-finding/instances-for-analysis :card 2)]- (is (= [c1 c2] (mapv :id batch))- "a sub-backlog batch serves the two oldest-analyzed stale cards, in ascending order"))))))))+(deftest ^:synchronized concurrent-upserts-test+ (testing "two analyses of an entity inserting its finding at once leave the finding they found, not an insert error"+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-temp [:model/Card {card-id :id} {}]+ (let [barrier (CyclicBarrier. 2)+ insert! (mt/original-fn #'t2/insert-returning-pk!)]+ (mt/with-dynamic-fn-redefs [t2/insert-returning-pk! (fn [model & args]+ (when (= :model/AnalysisFinding model)+ (.await barrier 10 TimeUnit/SECONDS))+ (apply insert! model args))]+ (let [upserts (repeatedly 2 #(future (models.analysis-finding/upsert-analysis! :card card-id true #{})))]+ (run! deref (doall upserts))))+ (is (=? {:result true} (finding card-id)))+ (is (zero? (t2/count :model/AnalysisFindingError :analyzed_entity_type "card"+ :analyzed_entity_id card-id))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljindex a9a5ae489e8..6324be20e89 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -47,43 +47,44 @@(deftest ^:synchronized card-deps-maintenance-test-1-new-card(testing "upstream deps of a card are updated correctly"- (mt/dataset test-data- (mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]- (let [card1 (card/create-card! (basic-orders) user)]- (deps.test/synchronously-run-backfill!)- (is (integer? (:id card1)))- (testing "when creating a new card"- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}- (upstream-of :card (:id card1))))- (testing "that depends on another card"- (let [card2 (card/create-card! (wrap-card card1) user)]- (deps.test/synchronously-run-backfill!)- (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}- (upstream-of :card (:id card2))))- (testing "but that doesn't affect the upstream deps of the inner card"- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}- (upstream-of :card (:id card1))))))))- (testing "when updating an existing card"- (testing "to add a new table dep"- (card/update-card! {:card-before-update card1- :card-updates- {:dataset_query (mt/mbql-query orders- {:joins [{:alias "Products"- :source-table (mt/id :products)- :condition [:= $id &Products.$products.id]}]})}})- (deps.test/synchronously-run-backfill!)- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))- (depends-on-> :card (:id card1) :table (mt/id :products))}- (upstream-of :card (:id card1)))))- (testing "to remove a table dep"- (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))- :card-updates- {:dataset_query (mt/mbql-query products)}})- (deps.test/synchronously-run-backfill!)- (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}- (upstream-of :card (:id card1)))))))))))))+ (mt/test-helpers-set-global-values!+ (mt/dataset test-data+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]+ (let [card1 (card/create-card! (basic-orders) user)]+ (deps.test/synchronously-run-dependency-job!)+ (is (integer? (:id card1)))+ (testing "when creating a new card"+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}+ (upstream-of :card (:id card1))))+ (testing "that depends on another card"+ (let [card2 (card/create-card! (wrap-card card1) user)]+ (deps.test/synchronously-run-dependency-job!)+ (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}+ (upstream-of :card (:id card2))))+ (testing "but that doesn't affect the upstream deps of the inner card"+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}+ (upstream-of :card (:id card1))))))))+ (testing "when updating an existing card"+ (testing "to add a new table dep"+ (card/update-card! {:card-before-update card1+ :card-updates+ {:dataset_query (mt/mbql-query orders+ {:joins [{:alias "Products"+ :source-table (mt/id :products)+ :condition [:= $id &Products.$products.id]}]})}})+ (deps.test/synchronously-run-dependency-job!)+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))+ (depends-on-> :card (:id card1) :table (mt/id :products))}+ (upstream-of :card (:id card1)))))+ (testing "to remove a table dep"+ (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))+ :card-updates+ {:dataset_query (mt/mbql-query products)}})+ (deps.test/synchronously-run-dependency-job!)+ (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}+ (upstream-of :card (:id card1))))))))))))))(deftest ^:synchronized card-deps-graph-test-1-mbql-card-chain(testing "deps graph is connected properly for a chain of MBQL cards"@@ -94,7 +95,7 @@(let [{id1 :id :as card1} (card/create-card! (basic-orders) user){id2 :id :as card2} (card/create-card! (wrap-card card1) user){id3 :id :as card3} (card/create-card! (wrap-card card2) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "raw deps are recorded correctly"(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))@@ -133,7 +134,7 @@{id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user){id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user){id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "raw deps are recorded correctly"(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))@@ -176,52 +177,52 @@{:aggregation [[:metric (:id metric-card)]]}):visualization_settings {}}user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "raw deps are recorded correctly for question using metric"(is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))(depends-on-> :card (:id question-card) :card (:id metric-card))}(upstream-of :card (:id question-card))))))))))))-(deftest dependency-status-marked-stale-test- (testing "dependency_status is marked stale when entities are updated, and not touched when they are created"- (mt/with-model-cleanup [:model/DependencyStatus]- (testing "cards"- (mt/with-model-cleanup [:model/Dependency]- (mt/with-temp [:model/Card card {}]- (mt/with-premium-features #{:dependencies}- (card/update-card! {:card-before-update card- :card-updates {:dataset_query (mt/mbql-query orders)}}))- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true)))))- (testing "transforms create"- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Transform transform]- (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform)))))))- (testing "transforms update"- (mt/with-premium-features #{}- (mt/with-temp [:model/Transform transform]- (mt/with-premium-features #{:dependencies}- (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))- (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))- (testing "snippets create"+(def ^:private pending? deps.test/dependency-pending?)++(def ^:private computed! deps.test/record-dependencies-computed!)++(deftest ^:synchronized dependency-status-marks-test+ (testing "updating what the dependency job reads of an entity marks its status stale, and creating one writes no+ status"+ (mt/test-helpers-set-global-values!+ (mt/with-model-cleanup [:model/DependencyStatus :model/Dependency](mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/NativeQuerySnippet snippet]- (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet)))))))- (testing "snippets update"- (mt/with-premium-features #{}- (mt/with-temp [:model/NativeQuerySnippet snippet]- (mt/with-premium-features #{:dependencies}- (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))+ (testing "cards"+ (mt/with-temp [:model/Card card {}]+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))+ (computed! :card (:id card))+ (card/update-card! {:card-before-update card+ :card-updates {:dataset_query (mt/mbql-query orders)}})+ (is (pending? :card (:id card)))))+ (testing "transforms"+ (mt/with-temp [:model/Transform transform]+ (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform))))+ (computed! :transform (:id transform))+ (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}})+ (is (pending? :transform (:id transform)))))+ (testing "snippets"+ (mt/with-temp [:model/NativeQuerySnippet snippet]+ (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet))))+ (computed! :snippet (:id snippet))+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"})+ (is (pending? :snippet (:id snippet))))))))))(deftest dependency-status-absent-until-backfill-on-card-create-test- (testing "a created card has no dependency_status row until the backfill job processes it"+ (testing "a created card has no dependency_status row until the dependency job processes it"(mt/with-premium-features #{:dependencies}(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user](let [card (card/create-card! (basic-orders) user)](is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))- (deps.test/synchronously-run-backfill!)- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale false))))))))+ (deps.test/synchronously-run-dependency-job!)+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card)))+ (is (not (pending? :card (:id card))))))))))(deftest filtered-graph-dependencies-test(testing "filtered-graph-dependencies respects filter clause"@@ -231,7 +232,7 @@(let [{id1 :id :as card1} (card/create-card! (basic-orders) user){id2 :id :as card2} (card/create-card! (wrap-card card1) user){id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "without filter, returns all dependencies"(let [graph (deps.graph/graph-dependencies)deps (graph/transitive graph [[:card id3]])]@@ -258,7 +259,7 @@(let [{id1 :id :as card1} (card/create-card! (basic-orders) user){id2 :id :as card2} (card/create-card! (wrap-card card1) user){id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "without filter, returns all dependents"(let [graph (deps.graph/graph-dependents)deps (graph/transitive graph [[:card id1]])]@@ -290,7 +291,7 @@:to_entity_type :table:to_entity_id (:id table1)})(deps.graph/swap-dependency! :card (:id card1) [:table (:id table1)] [:table (:id table2)])- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (not (t2/exists? :model/Dependency:from_entity_type :card:from_entity_id (:id card1)@@ -315,7 +316,7 @@:to_entity_id (:id table2)}]);; This would fail with duplicate key error before the fix(deps.graph/swap-dependency! :card (:id card2) [:table (:id table1)] [:table (:id table2)])- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (not (t2/exists? :model/Dependency:from_entity_type :card:from_entity_id (:id card2)diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.cljnew file mode 100644index 00000000000..512a18bfa9c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/result_metadata_test.clj@@ -0,0 +1,752 @@+(ns metabase-enterprise.dependencies.result-metadata-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.dependencies.calculation :as deps.calculation]+ [metabase-enterprise.dependencies.db :as dependencies.db]+ [metabase-enterprise.dependencies.result-metadata :as deps.result-metadata]+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]+ [metabase-enterprise.dependencies.test-util :as deps.test]+ [metabase-enterprise.test :as met]+ [metabase.batch-processing.core :as grouper]+ [metabase.config.core :as config]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.queries.core :as queries]+ [metabase.query-processor :as qp]+ [metabase.query-processor.db :as qp.db]+ [metabase.test :as mt]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private table-query deps.test/table-query)++(def ^:private card-query deps.test/card-query)++(def ^:private pending? deps.test/result-metadata-pending?)++(defn- do-with-inference+ "Run `thunk` with the job's inference of a card done by `f`, called with the real inference, the query and the card."+ [f thunk]+ (let [infer (mt/original-fn #'queries/infer-metadata-with-model-overrides)]+ (mt/with-dynamic-fn-redefs [queries/infer-metadata-with-model-overrides (fn [query card & _] (f infer query card))]+ (thunk))))++(defn- measure-query [measure-id]+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib.metadata/measure mp measure-id)))))++(defn- native-card-query [card-id]+ (let [tag (str "#" card-id)]+ (-> (lib/native-query (mt/metadata-provider) (str "SELECT * FROM {{" tag "}}"))+ (lib/with-template-tags {tag {:type :card+ :card-id card-id+ :name tag+ :display-name tag+ :id (str (random-uuid))}}))))++(defn- run-jobs!+ "Run the dependency job, whose current dependencies the result metadata job waits for, then the result metadata job,+ as each triggers the other, until neither has anything left to do."+ []+ (mt/with-premium-features #{:dependencies}+ (loop [runs 1]+ (deps.test/synchronously-run-dependency-job!)+ (when (and (pos? (deps.result-metadata/update-result-metadata! 1000)) (< runs 20))+ (recur (inc runs))))))++(defn- column [card-id column-name]+ (some #(when (= column-name (:name %)) %)+ (t2/select-one-fn :result_metadata :model/Card :id card-id)))++(defn- rename-column [metadata column-name display-name]+ (mapv #(cond-> % (= column-name (:name %)) (assoc :display_name display-name)) metadata))++(defmacro ^:private with-dependencies-feature+ "Run `body` with the dependencies feature, each write committing on its own as it does between real requests."+ [& body]+ `(mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ ~@body)))++(defn- reprocess!+ "Have the result metadata job process the Cards with `card-ids` again, as if they changed."+ [& card-ids]+ (dependencies.db/delete-result-metadata-statuses! (vec card-ids)))++(defn- rederive!+ "Run one batch of the result metadata job."+ []+ (mt/with-premium-features #{:dependencies}+ (deps.result-metadata/update-result-metadata! 1000)))++(defn- add-grand-total [query]+ (lib/expression query "Grand total" (lib/* (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total)) 2)))++(deftest model-query-edit-test+ (testing "editing a model's query re-derives the MBQL cards depending on it transitively, but not past a native card"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}+ :model/Card {question-id :id} {:dataset_query (card-query model-id)}+ :model/Card {nested-id :id} {:dataset_query (card-query question-id)}+ :model/Card {native-id :id} {:dataset_query (native-card-query model-id)}+ :model/Card {after-native-id :id} {:dataset_query (card-query native-id)}]+ (run-jobs!)+ (is (nil? (column question-id "Grand total")))+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})+ (is (pending? question-id))+ (run-jobs!)+ (is (some? (column question-id "Grand total")))+ (is (some? (column nested-id "Grand total")))+ (testing "the card depending on the native card is left alone"+ (is (nil? (column after-native-id "Grand total"))))+ (is (not-any? pending? [model-id question-id nested-id]))+ (testing "a column no longer inferred is removed"+ (t2/update! :model/Card model-id {:dataset_query (table-query :orders)})+ (run-jobs!)+ (is (nil? (column question-id "Grand total")))+ (is (nil? (column nested-id "Grand total"))))))))++(deftest run-metadata-test+ (testing "re-deriving a card keeps the fingerprints and types its run stored for the columns it still has"+ (with-dependencies-feature+ (let [mp (mt/metadata-provider)+ query (-> (table-query :orders)+ (lib/aggregate (lib/count))+ (lib/breakout (lib.metadata/field mp (mt/id :orders :product_id))))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]+ (mt/with-test-user :crowberto+ (qp/process-query (qp/userland-query query {:card-id card-id})))+ (let [run-count (column card-id "count")]+ (is (some? (:fingerprint run-count)))+ (run-jobs!)+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :product_id) :display_name "Item"}]+ (run-jobs!)+ (is (= "Item" (:display_name (column card-id "PRODUCT_ID"))))+ (is (= (select-keys run-count [:base_type :fingerprint])+ (select-keys (column card-id "count") [:base_type :fingerprint]))))))))))++(deftest fingerprints-come-only-from-stored-metadata-test+ (testing "re-deriving a card keeps the stored fingerprints and adds none for a column the stored metadata lacks"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (let [marker {:global {:distinct-count 7}}+ stored (->> (t2/select-one-fn :result_metadata :model/Card :id card-id)+ (remove #(= "PRICE" (:name %)))+ (mapv #(cond-> % (= "ID" (:name %)) (assoc :fingerprint marker))))]+ (t2/update! :model/Card card-id {:result_metadata stored})+ (run-jobs!)+ (reprocess! card-id)+ (run-jobs!)+ (is (some? (column card-id "PRICE")))+ (is (nil? (:fingerprint (column card-id "PRICE"))))+ (is (= marker (:fingerprint (column card-id "ID")))))))))++(deftest native-parent-test+ (testing "a card depending on a changed model directly is re-derived even when it also depends on a native card"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}+ :model/Card {native-id :id} {:dataset_query (native-card-query model-id)+ :result_metadata (t2/select-one-fn :result_metadata :model/Card+ :id model-id)}]+ (let [mp (mt/metadata-provider)+ query (card-query model-id)+ native (lib.metadata/card mp native-id)+ joined (lib/join query (lib/join-clause native [(lib/= (lib/ref (lib.metadata/field mp (mt/id :orders :id)))+ (lib/ref (lib.metadata/field mp (mt/id :orders :id))))]))]+ (mt/with-temp [:model/Card {joined-id :id} {:dataset_query joined}]+ (run-jobs!)+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})+ (run-jobs!)+ (is (some? (column joined-id "Grand total")))))))))++(deftest internal-card-test+ (testing "the job leaves the result metadata of cards Metabase created itself, such as Usage Analytics, alone"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query model-id)}+ :model/Card {internal-id :id} {:dataset_query (card-query model-id)+ :creator_id config/internal-mb-user-id}]+ (run-jobs!)+ (t2/update! :model/Card model-id {:dataset_query (add-grand-total (table-query :orders))})+ (run-jobs!)+ (is (some? (column card-id "Grand total")))+ (is (nil? (column internal-id "Grand total")))))))++(deftest metadata-edit-test+ (testing "editing a model's column overrides re-derives the cards on the model, which take them"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query model-id)}]+ (run-jobs!)+ (t2/update! :model/Card model-id {:result_metadata (rename-column (t2/select-one-fn :result_metadata :model/Card+ :id model-id)+ "TOTAL" "Grand total")})+ (is (pending? card-id))+ (run-jobs!)+ (is (= "Grand total" (:display_name (column card-id "TOTAL"))))))))++(deftest model-overrides-test+ (testing "re-deriving a model after its table changed keeps the model's column overrides"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query (table-query :orders)}]+ (run-jobs!)+ (t2/update! :model/Card model-id {:result_metadata (rename-column (t2/select-one-fn :result_metadata :model/Card+ :id model-id)+ "TOTAL" "Grand total")})+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]+ (is (pending? model-id))+ (run-jobs!)+ (is (not (pending? model-id)))+ (is (= "Grand total" (:display_name (column model-id "TOTAL")))))))))++(deftest field-user-settings-test+ (testing "renaming a field re-derives the MBQL cards reading its table"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Grand total"}]+ (is (pending? card-id))+ (run-jobs!)+ (is (= "Grand total" (:display_name (column card-id "TOTAL"))))+ (is (not (pending? card-id))))))))++(deftest measure-edit-test+ (testing "changing a measure's definition re-derives a card using it"+ (with-dependencies-feature+ (mt/with-temp [:model/Measure {measure-id :id} {:name "Measure"+ :table_id (mt/id :orders)+ :creator_id (mt/user->id :rasta)+ :definition (-> (table-query :orders) (lib/aggregate (lib/count)))}+ :model/Card {card-id :id} {:dataset_query (measure-query measure-id)}]+ (run-jobs!)+ (let [column-name #(:name (first (t2/select-one-fn :result_metadata :model/Card :id card-id)))]+ (is (= "count" (column-name)))+ (t2/update! :model/Measure measure-id+ {:definition (-> (table-query :orders)+ (lib/aggregate (lib/avg (lib.metadata/field (mt/metadata-provider)+ (mt/id :orders :total)))))})+ (run-jobs!)+ (is (= "avg" (column-name))))))))++(deftest measure-chain-edit-test+ (testing "changing a measure's definition re-derives a card using a measure built on it"+ (with-dependencies-feature+ (mt/with-temp [:model/Measure {inner-id :id} {:name "Inner"+ :table_id (mt/id :orders)+ :creator_id (mt/user->id :rasta)+ :definition (-> (table-query :orders) (lib/aggregate (lib/count)))}]+ (mt/with-temp [:model/Measure {outer-id :id} {:name "Outer"+ :table_id (mt/id :orders)+ :creator_id (mt/user->id :rasta)+ :definition (measure-query inner-id)}+ :model/Card {card-id :id} {:dataset_query (measure-query outer-id)}]+ (run-jobs!)+ (let [base-type #(:base_type (first (t2/select-one-fn :result_metadata :model/Card :id card-id)))]+ (is (isa? (base-type) :type/Integer))+ (t2/update! :model/Measure inner-id+ {:definition (-> (table-query :orders)+ (lib/aggregate (lib/avg (lib.metadata/field (mt/metadata-provider)+ (mt/id :orders :total)))))})+ (run-jobs!)+ (is (isa? (base-type) :type/Float))))))))++(defn- rederive-during-inference!+ "Re-derive the Card with `card-id` while `write!` runs in the middle of its inference, returning how many entities the+ job took."+ [card-id write!]+ (reprocess! card-id)+ (do-with-inference (fn [infer query card]+ (write!)+ (infer query card))+ rederive!))++(defn- store-run-metadata!+ "Store `metadata` as the result metadata of the Card with `card-id`, as a run of the card does."+ [card-id metadata]+ (#'qp.db/set-card-result-metadata! card-id metadata))++(defn- once!+ "A function running `f` the first time it is called and doing nothing after."+ [f]+ (let [called (atom false)]+ (fn [] (when (compare-and-set! called false true) (f)))))++(deftest user-save-during-rederive-test+ (testing "a user's save of a card's query during its re-derive is kept, and the card is re-derived again from it"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]+ (is (pos? (rederive-during-inference!+ card-id+ (once! #(t2/update! :model/Card card-id {:dataset_query (add-grand-total (table-query :orders))})))))+ (is (some? (column card-id "Grand total")))+ (is (pending? card-id))+ (run-jobs!)+ (is (= "Order total" (:display_name (column card-id "TOTAL"))))+ (is (some? (column card-id "Grand total")))+ (is (not (pending? card-id))))))))++(deftest run-write-during-rederive-test+ (testing "a run storing a card's metadata during its re-derive is kept, and the card is re-derived again from it"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (let [run-metadata (-> (t2/select-one-fn :result_metadata :model/Card :id card-id)+ (rename-column "TOTAL" "Run total")+ (->> (mapv #(cond-> % (= "ID" (:name %)) (assoc :fingerprint {:global {:distinct-count 7}})))))]+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]+ (is (pos? (rederive-during-inference! card-id (once! #(store-run-metadata! card-id run-metadata)))))+ (run-jobs!)+ (is (= "Order total" (:display_name (column card-id "TOTAL"))))+ (is (= {:global {:distinct-count 7}} (:fingerprint (column card-id "ID"))))))))))++(deftest card-on-card-temporal-unit-test+ (testing "a card on a card takes the new temporal unit with the new display name"+ (with-dependencies-feature+ (let [mp (mt/metadata-provider)+ by-unit #(-> (table-query :orders)+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket (lib.metadata/field mp (mt/id :orders :created_at)) %)))]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (by-unit :month)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (t2/update! :model/Card source-id {:dataset_query (by-unit :year)})+ (run-jobs!)+ (is (=? {:display_name "Created At: Year" :unit :year}+ (column card-id "CREATED_AT"))))))))++(deftest card-on-card-type-test+ (testing "a card on a card takes the new type of a column whose type changed upstream"+ (with-dependencies-feature+ (let [total (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total))+ with #(lib/expression (table-query :orders) "X" %)]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (with (lib/* total 2))}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (is (isa? (:base_type (column card-id "X")) :type/Number))+ (t2/update! :model/Card source-id {:dataset_query (with (lib/concat total "!"))})+ (run-jobs!)+ (is (= :type/Text (:base_type (column card-id "X"))))+ (testing "within one root type too, since no run measured the card's types"+ (t2/update! :model/Card source-id {:dataset_query (with (lib/+ (lib.metadata/field (mt/metadata-provider)+ (mt/id :orders :quantity))+ 1))})+ (run-jobs!)+ (is (= :type/Integer (:base_type (column card-id "X"))))+ (t2/update! :model/Card source-id {:dataset_query (with (lib/* total 1.5))})+ (run-jobs!)+ (is (= :type/Float (:base_type (column card-id "X"))))))))))++(deftest alias-less-stored-metadata-test+ (testing "stored columns without aliases are matched by name, so an alias equal to another column's name merges+ nothing"+ (let [stored [{:name "CREATED_AT" :id 36 :base_type "type/DateTime" :database_type "TIMESTAMP"+ :fingerprint {:global {:distinct-count 1}}}+ {:name "CREATED_AT_2" :id 14 :base_type "type/DateTime" :database_type "TIMESTAMP"+ :fingerprint {:global {:distinct-count 2}}}]+ inferred [{:name "CREATED_AT" :id 36 :base_type :type/DateTime+ :lib/desired-column-alias "PRODUCTS__via__PRODUCT_ID__CREATED_AT"}+ {:name "CREATED_AT_2" :id 14 :base_type :type/DateTime :lib/desired-column-alias "CREATED_AT"}]]+ (is (=? [{:id 36 :fingerprint {:global {:distinct-count 1}}}+ {:id 14 :fingerprint {:global {:distinct-count 2}}}]+ (#'deps.result-metadata/patch-result-metadata stored inferred))))))++(deftest kebab-case-stored-metadata-test+ (testing "stored columns with kebab-case keys compare like snake_case ones"+ (is (= [{:semantic_type "type/FK" :fk_target_field_id 29}]+ (#'deps.result-metadata/comparable-metadata+ (mapv #'deps.result-metadata/normalize-column [{:semantic-type :type/FK :fk-target-field-id 29}]))))))++(deftest failure-waits-for-change-test+ (testing "a card whose re-derive keeps failing is given up on, not re-derived again until it or an upstream changes+ again, and then is"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (do-with-inference (fn [_infer _query _card] (throw (ex-info "Inference failed" {})))+ run-jobs!)+ (is (nil? (column card-id "Grand total")))+ (is (not (pending? card-id)))+ (run-jobs!)+ (is (nil? (column card-id "Grand total")))+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :total) :display_name "Order total"}]+ (run-jobs!)+ (is (some? (column card-id "Grand total"))))))))++(deftest result-metadata-job-before-dependency-job-test+ (testing "a card whose dependencies the dependency job computes after its upstream changed is re-derived once it does"+ (with-dependencies-feature+ (mt/with-model-cleanup [:model/Card]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (let [card-id (t2/insert-returning-pk! :model/Card {:name "Dependent"+ :display :table+ :visualization_settings {}+ :creator_id (mt/user->id :rasta)+ :database_id (mt/id)+ :dataset_query (card-query source-id)})]+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (mt/with-premium-features #{:dependencies}+ (deps.result-metadata/update-result-metadata! 1000))+ (run-jobs!)+ (is (some? (column card-id "Grand total")))))))))++(deftest new-edge-to-upstream-processed-before-save-test+ (testing "a new dependency on a card processed before the dependent was saved has the card itself not processed again"+ (with-dependencies-feature+ (mt/with-model-cleanup [:model/Card]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card _ {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (is (not (pending? source-id)))+ (Thread/sleep 5)+ (t2/insert! :model/Card {:name "Dependent"+ :display :table+ :visualization_settings {}+ :creator_id (mt/user->id :rasta)+ :database_id (mt/id)+ :dataset_query (card-query source-id)})+ (deps.test/synchronously-run-dependency-job!)+ (is (not (pending? source-id))))))))++(deftest new-dependent-of-upstream-processed-before-its-edges-test+ (testing "a card whose upstream changed and was processed before the card's dependencies existed is re-derived when+ they are computed, even when the card was saved again meanwhile"+ (with-dependencies-feature+ (mt/with-model-cleanup [:model/Card]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (let [card-id (t2/insert-returning-pk! :model/Card {:name "Dependent"+ :display :table+ :visualization_settings {}+ :creator_id (mt/user->id :rasta)+ :database_id (mt/id)+ :dataset_query (card-query source-id)})]+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (deps.result-metadata/update-result-metadata! 1000)+ (t2/update! :model/Card card-id {:name "Renamed"})+ (run-jobs!)+ (is (some? (column card-id "Grand total")))))))))++(deftest job-reruns-while-it-takes-entities-test+ (testing "the job runs again right away while it took entities, since a card it stored changed in turn"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card _ {:dataset_query (card-query source-id)}]+ (deps.test/synchronously-run-dependency-job!)+ (is (= 1 (#'task.result-metadata/run-batch!)))+ (run-jobs!)+ (is (nil? (#'task.result-metadata/run-batch!)))))))++(deftest storing-card-triggers-jobs-test+ (testing "the job stores a re-derived card through the Card model, so its hooks trigger the jobs, and keeps its+ updated_at"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (deps.test/synchronously-run-dependency-job!)+ (let [submitted (atom [])+ before (t2/select-one-fn :updated_at :model/Card :id card-id)]+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (deps.test/synchronously-run-dependency-job!)+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))]+ (rederive!)+ (rederive!))+ (is (some? (column card-id "Grand total")))+ (is (= before (t2/select-one-fn :updated_at :model/Card :id card-id)))+ (is (seq @submitted)))))))++(deftest sandboxed-creator-test+ (testing "a card is re-derived as no user, so its creator's sandbox doesn't decide its columns, and the columns its+ stored metadata lacks get no fingerprint"+ (met/with-gtaps! {:gtaps {:venues {:query (let [mp (mt/metadata-provider)]+ (lib/with-fields (table-query :venues)+ [(lib.metadata/field mp (mt/id :venues :id))+ (lib.metadata/field mp (mt/id :venues :name))+ (lib.metadata/field mp (mt/id :venues :category_id))]))}}}+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)+ :creator_id (mt/user->id :rasta)+ :result_metadata []}]+ (deps.test/synchronously-run-dependency-job!)+ (reprocess! card-id)+ (mt/with-premium-features #{:dependencies :sandboxes}+ (deps.result-metadata/update-result-metadata! 1000))+ (let [columns (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (= ["ID" "NAME" "CATEGORY_ID" "LATITUDE" "LONGITUDE" "PRICE"] (mapv :name columns)))+ (is (every? (comp nil? :fingerprint) columns))))))))++(deftest legacy-stored-card-test+ (testing "a card whose stored query is legacy MBQL with a filter is stored once and the job stops, though each read+ gives the converted query new uuids"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (t2/update! (t2/table-name :model/Card) card-id+ {:dataset_query (json/encode {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :filter [:> [:field (mt/id :venues :price) nil] 1]}})+ :result_metadata (json/encode [])})+ (run-jobs!)+ (is (seq (column card-id "PRICE")))+ (is (not (pending? card-id)))+ (is (nil? (#'task.result-metadata/run-batch!)))))))++(deftest new-upstream-changed-before-its-edge-test+ (testing "a card changed to read a card that changes before the dependency job stores the new edge is re-derived+ from the changed one"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (t2/update! :model/Card card-id {:dataset_query (card-query source-id)})+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (run-jobs!)+ (is (some? (column card-id "Grand total")))))))++(deftest failing-store-test+ (testing "a store that throws doesn't stop the other cards of the batch"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {failing-id :id} {:dataset_query (table-query :venues)}+ :model/Card {other-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! failing-id other-id)+ (let [store (mt/original-fn #'dependencies.db/set-card-result-metadata!)]+ (mt/with-dynamic-fn-redefs [dependencies.db/set-card-result-metadata!+ (fn [card-id metadata]+ (if (= failing-id card-id)+ (throw (ex-info "Store failed" {}))+ (store card-id metadata)))]+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed"))+ rederive!)))+ (is (= "Changed" (:display_name (column other-id "NAME"))))))))++(deftest own-edit-rederives-dependents-test+ (testing "editing a card's query re-derives the cards depending on it, not the card, whose save inferred it"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (let [inferred (atom #{})]+ (do-with-inference (fn [infer query card]+ (swap! inferred conj (:id card))+ (infer query card))+ run-jobs!)+ (is (= #{card-id} @inferred))+ (is (some? (column card-id "Grand total"))))))))++(defn- native-run-metadata [sql]+ (get-in (qp/process-query (lib/native-query (mt/metadata-provider) sql)) [:data :results_metadata :columns]))++(deftest native-card-result-metadata-test+ (testing "a write of a native card's run metadata re-derives its dependents"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {native-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT ID, NAME FROM VENUES")+ :result_metadata (native-run-metadata "SELECT ID, NAME FROM VENUES")}+ :model/Card {card-id :id} {:dataset_query (card-query native-id)}]+ (run-jobs!)+ (t2/update! :model/Card native-id {:dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT ID, NAME, PRICE FROM VENUES")})+ (run-jobs!)+ (is (nil? (column card-id "PRICE")))+ (store-run-metadata! native-id (native-run-metadata "SELECT ID, NAME, PRICE FROM VENUES"))+ (is (pending? native-id))+ (run-jobs!)+ (is (some? (column card-id "PRICE")))+ (is (not (pending? card-id)))))))++(deftest measured-keys-test+ (testing "a re-derived column keeps what a run measured only while its root type and source type are unchanged, and only the stored fingerprint"+ (let [stored {:name "X" :base_type "type/BigInteger" :database_type "BIGINT" :fingerprint {:type {:type/Number {}}}}+ patch #(first (#'deps.result-metadata/patch-result-metadata [stored] [%]))]+ (testing "a run's refinement within the root type and source type is kept"+ (is (=? {:base_type "type/BigInteger" :fingerprint {:type {:type/Number {}}}}+ (patch {:name "X" :base_type :type/Integer :database_type "BIGINT"}))))+ (testing "a new root type takes the inferred type and keeps the stored fingerprint"+ (is (= {:name "X" :base_type :type/Text :database_type "VARCHAR" :fingerprint {:type {:type/Number {}}}}+ (patch {:name "X" :base_type :type/Text :database_type "VARCHAR" :fingerprint {:type {:type/Text {}}}}))))+ (testing "a new source type within the root type takes the inferred type and keeps the stored fingerprint"+ (is (= {:name "X" :base_type :type/Decimal :database_type "DECIMAL" :fingerprint {:type {:type/Number {}}}}+ (patch {:name "X" :base_type :type/Decimal :database_type "DECIMAL"}))))+ (testing "a column without a stored one gets no fingerprint"+ (is (= {:name "Y" :base_type :type/Text}+ (first (#'deps.result-metadata/patch-result-metadata+ [stored] [{:name "Y" :base_type :type/Text :fingerprint {:type {:type/Text {}}}}]))))))))++(deftest remapped-column-test+ (testing "a card storing the metadata of a run with remapped columns is not rewritten, since inference adds them too"+ (mt/with-column-remappings [orders.product_id products.title]+ (let [query (table-query :orders)+ run (get-in (qp/process-query query) [:data :results_metadata :columns])]+ (is (some :remapped_from run))+ (mt/with-temp [:model/Card card {:dataset_query query :result_metadata run}]+ (is (nil? (#'deps.result-metadata/rederived-result-metadata card))))))))++(deftest failed-dependencies-block-rederive-test+ (testing "a card whose dependency computation keeps failing is not re-derived from its old edges until it changes"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (card-query source-id)}]+ (run-jobs!)+ (let [calculate (mt/original-fn #'deps.calculation/calculate-deps)]+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps+ (fn [entity-type entity]+ (if (= card-id (:id entity))+ (throw (ex-info "Computation failed" {}))+ (calculate entity-type entity)))]+ (t2/update! :model/Card card-id {:dataset_query (card-query source-id)+ :description "edited"})+ (dependencies.db/mark-dependency-statuses-stale! {:card #{card-id}})+ (t2/update! :model/Card source-id {:dataset_query (add-grand-total (table-query :orders))})+ (run-jobs!)))+ (is (nil? (column card-id "Grand total")))+ (is (pending? card-id))))))++(deftest crash-leaves-card-pending-test+ (testing "a job that dies between taking a card's status and clearing it leaves the card pending, and the next run+ re-derives it"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! card-id)+ (is (thrown? Error (do-with-inference (fn [_infer _query _card] (throw (Error. "crash"))) rederive!)))+ (is (pending? card-id))+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed")) rederive!)+ (is (= "Changed" (:display_name (column card-id "NAME"))))+ (is (not (pending? card-id)))))))++(deftest write-during-rederive-keeps-card-pending-test+ (testing "a write counted on a card while it is re-derived keeps it pending after the job clears what it read"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! card-id)+ (do-with-inference (fn [infer query card]+ (dependencies.db/mark-result-metadata-statuses-stale! {:card #{card-id}})+ (infer query card))+ rederive!)+ (is (pending? card-id))+ (rederive!)+ (is (not (pending? card-id)))))))++(deftest store-compares-same-read-test+ (testing "a card whose full read normalizes its stored metadata away is stored, since both compared reads are alike"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! card-id)+ (let [card (mt/original-fn #'dependencies.db/card)]+ (mt/with-dynamic-fn-redefs [dependencies.db/card (fn [id] (assoc (card id) :result_metadata nil))]+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "NAME" "Changed"))+ rederive!)))+ (is (= "Changed" (:display_name (column card-id "NAME"))))+ (is (not (pending? card-id)))))))++(deftest conflict-fails-test+ (testing "a card that keeps changing during its re-derive is left as the other writer stored it, and other cards proceed"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {orders-card-id :id} {:dataset_query (table-query :orders)}+ :model/Card {other-card-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (reprocess! orders-card-id other-card-id)+ (do-with-inference (fn [infer query card]+ (u/prog1 (rename-column (infer query card) "TOTAL" "Changed")+ (when (= orders-card-id (:id card))+ (store-run-metadata!+ orders-card-id+ (rename-column (:result_metadata card) "TOTAL" (str (random-uuid)))))))+ rederive!)+ (is (not= "Changed" (:display_name (column orders-card-id "TOTAL"))))+ (is (= "Changed" (:display_name (column other-card-id "TOTAL"))))))))++(deftest store-locks-only-dependents-test+ (testing "storing a re-derived card locks only the statuses of its dependents, so a write of another status meanwhile+ doesn't wait for it"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}+ :model/Card {other-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! card-id)+ (let [store (mt/original-fn #'dependencies.db/set-card-result-metadata!)+ marked (atom nil)]+ (mt/with-dynamic-fn-redefs [dependencies.db/set-card-result-metadata!+ (fn [& args]+ (reset! marked (deref (future (t2/update! :model/ResultMetadataStatus+ :entity_type :card+ :entity_id other-id+ {:stale 0}))+ 5000 :timed-out))+ (apply store args))]+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "TOTAL" "Changed"))+ rederive!))+ (is (= 1 @marked))+ (is (= "Changed" (:display_name (column card-id "TOTAL")))))))))++(deftest store-waits-for-save-in-flight-test+ (testing "a save of the card holding its row lock while the job stores has the job re-derive it after the save"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :orders)}]+ (run-jobs!)+ (reprocess! card-id)+ (let [locked (promise)+ save (future+ (t2/with-transaction [_]+ (t2/update! :model/Card card-id {:name "Saved"})+ (deliver locked true)+ (Thread/sleep 1000)))]+ @locked+ (do-with-inference (fn [infer query card] (rename-column (infer query card) "TOTAL" "Changed"))+ #(mt/with-premium-features #{:dependencies}+ (deps.result-metadata/update-result-metadata! 1000)+ @save+ (deps.test/synchronously-run-dependency-job!)+ (deps.result-metadata/update-result-metadata! 1000)))+ (is (= "Saved" (t2/select-one-fn :name :model/Card :id card-id)))+ (is (= "Changed" (:display_name (column card-id "TOTAL")))))))))++(deftest failing-dependent-test+ (testing "a card whose re-derive fails doesn't stop its siblings, and stays pending for the next run"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {first-id :id} {:dataset_query (table-query :venues)}+ :model/Card {failing-id :id} {:dataset_query (table-query :venues)}+ :model/Card {last-id :id} {:dataset_query (table-query :venues)}]+ (run-jobs!)+ (reprocess! first-id failing-id last-id)+ (do-with-inference (fn [infer query card]+ (if (= failing-id (:id card))+ (throw (ex-info "Inference failed" {}))+ (rename-column (infer query card) "NAME" "Changed")))+ rederive!)+ (is (= ["Changed" "Changed"] (mapv #(:display_name (column % "NAME")) [first-id last-id])))+ (is (pending? failing-id))))))++(deftest store-updates-parameters-using-dropped-columns-test+ (testing "storing a re-derived card resets the parameters taking values from a column it no longer has"+ (with-dependencies-feature+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (table-query :venues)}+ :model/Card {card-id :id} {:dataset_query (table-query :venues)+ :parameters [{:id "price"+ :type :category+ :name "Price"+ :slug "price"+ :values_source_type :card+ :values_source_config {:card_id source-id+ :value_field [:field (mt/id :venues :price) nil]}}]}]+ (run-jobs!)+ (reprocess! source-id)+ (do-with-inference (fn [infer query card]+ (cond->> (infer query card)+ (= source-id (:id card)) (into [] (remove #(= "PRICE" (:name %))))))+ rederive!)+ (is (nil? (column source-id "PRICE")))+ (is (=? [{:id "price" :values_source_type nil}]+ (t2/select-one-fn :parameters :model/Card :id card-id)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.cljindex 956d3f0e5bc..deb1e3bbfb3 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj@@ -2,157 +2,120 @@(:require[clojure.java.jdbc :as jdbc][clojure.test :refer [deftest is testing]]- [java-time.api :as t]- [metabase-enterprise.dependencies.db :as deps.db]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.hooks]+ [metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.sync.core :as sync][metabase.test :as mt][metabase.test.data.one-off-dbs :as one-off-dbs]- [metabase.util :as u][toucan2.core :as t2]))+(defn- pending?+ "Whether the Card with `card-id` is to be re-analyzed and re-derived."+ [card-id]+ (and (deps.test/analysis-finding-pending? :card card-id)+ (deps.test/result-metadata-pending? card-id)))++(defn- record-processed!+ "Record that the analysis and result metadata jobs processed the Card with `card-id`."+ [card-id]+ (deps.test/record-analysis-finding-processed! :card card-id)+ (deps.test/record-result-metadata-processed! card-id))+(defn- with-syncable-db! [thunk]- (mt/with-premium-features #{:dependencies}- (one-off-dbs/with-blank-db- ;; Step 1: Create a table with two columns- (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"- "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]- (jdbc/execute! one-off-dbs/*conn* [statement]))- ;; Sync the database to pick up the new table- (sync/sync-database! (mt/db))- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")- filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")- ;; Step 2: Create a card with a filter on filter_col- mp (lib-be/application-database-metadata-provider (mt/id))- table (lib.metadata/table mp table-id)- filter-field (lib.metadata/field mp filter-field-id)- query (-> (lib/query mp table)- (lib/filter (lib/= filter-field "value1")))]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query- :database_id (mt/id)}]- ;; Create the dependency manually- (t2/insert! :model/Dependency {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id table-id})- ;; Step 3: Run analysis - should succeed with no errors- (let [card (t2/select-one :model/Card card-id)]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:dependencies}+ (one-off-dbs/with-blank-db+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]+ (jdbc/execute! one-off-dbs/*conn* [statement]))+ (sync/sync-database! (mt/db))+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")+ mp (lib-be/application-database-metadata-provider (mt/id))+ query (-> (lib/query mp (lib.metadata/table mp table-id))+ (lib/filter (lib/= (lib.metadata/field mp filter-field-id) "value1")))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query+ :database_id (mt/id)}]+ (t2/insert! :model/Dependency {:from_entity_type :card+ :from_entity_id card-id+ :to_entity_type :table+ :to_entity_id table-id});; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see;; old values after the sync!(lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! card)))- (testing "Initial analysis succeeds"- (is (true?- (t2/select-one-fn :result :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)))- (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))- ;; Backdate the analysis timestamp and table/field updated_at so- ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)- ;; after a new sync makes an edit.- ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and- ;; the field update from sync would have identical timestamps.- (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]- (t2/update! :model/AnalysisFinding- {:analyzed_entity_type :card :analyzed_entity_id card-id}- {:analyzed_at old-time})- (t2/update! :model/Field :table_id table-id- {:updated_at old-time})- (t2/update! :model/Table :id table-id- {:updated_at old-time}))- ;; Setup complete: Now call the thunk for the actual test run.- (thunk {:db-id (:id (mt/db))- :card-id card-id- :table-id table-id- :filter-field-id filter-field-id}))))))+ (deps.analysis-finding/upsert-analysis! (t2/select-one :model/Card card-id)))+ (testing "Initial analysis succeeds"+ (is (true? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_type :card :entity_id card-id)))+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))+ (record-processed! card-id)+ (thunk {:card-id card-id+ :table-id table-id+ :filter-field-id filter-field-id})))))))-;; Integration test that a real DB sync triggers re-analysis of an updated table.(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test- (testing "When sync detects a removed column, re-analyzing the card shows errors"+ (testing "when sync deactivates a column, the cards on its table are re-analyzed and show the missing column"(with-syncable-db!(fn [{:keys [card-id filter-field-id]}]- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- ;; Step 4: Remove the column used in the filter- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])- ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis- ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.- (sync/sync-database! (mt/db))- ;; Verify the field is now inactive- (testing "Field is marked inactive after sync"- (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))- ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.- ;; Due to race conditions, the entity may be either stale (job hasn't run yet)- ;; or already reanalyzed (job ran). Check for either valid outcome.- (let [finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (is (some? finding) "Analysis finding should exist")- (if (:stale finding)- ;; Job hasn't run yet - sync marked it stale, that's the expected behavior- (testing "Entity is marked stale for re-analysis"- (is (true? (:stale finding))))- ;; Job already ran - verify the reanalysis detected the missing column- (testing "Re-analysis detects the missing column"- (is (false? (:result finding)) "Analysis should fail")- (is (not= initial-analysis-time (:analyzed_at finding))- "Analysis should have been re-run")- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]- (is (seq errors) "Should have analysis errors")- (is (some #(= :missing-column (:error_type %)) errors)- "Should have a missing-column error"))))))))))+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])+ (sync/sync-database! (mt/db))+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id)))+ (is (pending? card-id))+ (deps.test/synchronously-run-dependency-job!)+ (deps.test/run-analysis-finding-job!)+ (is (false? (t2/select-one-fn :result :model/AnalysisFinding+ :entity_type :card :entity_id card-id)))+ (is (some #(= :missing-column (:error_type %))+ (deps.analysis-finding-error/errors-for-entity :card card-id)))+ (is (not (deps.test/analysis-finding-pending? :card card-id)))))))(deftest ^:synchronized scheduled-metadata-sync-triggers-reanalysis-test- (testing "The metadata sync the scheduled sync task runs marks the dependents of a changed table for re-analysis"+ (testing "the metadata sync the scheduled sync task runs has the cards on a table it changes processed"(with-syncable-db!(fn [{:keys [card-id]}]- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])- (sync/sync-db-metadata! (mt/db))- (let [finding (t2/select-one :model/AnalysisFinding :analyzed_entity_type :card :analyzed_entity_id card-id)]- (is (or (:stale finding)- (not= initial-analysis-time (:analyzed_at finding))))))))))+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])+ (sync/sync-db-metadata! (mt/db))+ (is (pending? card-id))))))-;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all-;; cards which depend on that table.(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test- (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"+ (testing "a sync that changes nothing about a table or its fields has nothing processed"+ (with-syncable-db!+ (fn [{:keys [card-id filter-field-id table-id]}]+ (let [table-before (into {} (t2/select-one :model/Table :id table-id))+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]+ (sync/sync-database! (mt/db))+ (is (= table-before (into {} (t2/select-one :model/Table :id table-id))))+ (is (= filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))))+ (is (not (pending? card-id)))+ (is (not (deps.test/analysis-finding-pending? :card card-id))))))))++(deftest ^:synchronized table-sync-makes-cards-on-changed-table-pending-test+ (testing "a sync of one table, like after an upload or a transform run, has the cards on it processed only when it+ changes it"+ (with-syncable-db!+ (fn [{:keys [card-id table-id]}]+ (sync/sync-table! (t2/select-one :model/Table :id table-id))+ (is (not (pending? card-id)))+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" ADD COLUMN \"new_col\" VARCHAR;"])+ (sync/sync-table! (t2/select-one :model/Table :id table-id))+ (is (pending? card-id))))))++(deftest ^:synchronized sync-discovered-fk-makes-cards-on-its-table-pending-test+ (testing "a foreign key a sync discovers has the cards on the table of its field processed"(with-syncable-db!- (fn [{:keys [card-id db-id filter-field-id table-id]}]- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)- db-deps-checked (atom [])- original-deps-check (mt/original-fn #'deps.db/table-ids-with-outdated-findings)- table-before (into {} (t2/select-one :model/Table :id table-id))- filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]- ;; Re-sync the database, having made no changes to it.- (mt/with-dynamic-fn-redefs [deps.db/table-ids-with-outdated-findings- (fn [db-id]- (u/prog1 (original-deps-check db-id)- (swap! db-deps-checked conj [db-id <>])))]- (sync/sync-database! (mt/db)))- (testing "sync doesn't update tables or fields that haven't changed"- (let [table-after (into {} (t2/select-one :model/Table :id table-id))- filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]- (is (= table-before table-after))- (is (= filter-field-before filter-field-after))))- (testing "the DB is were checked for tables that need deps updates, but none were found"- (is (=? [[db-id empty?]]- @db-deps-checked)))- (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"- (let [finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (is (some? finding) "Analysis finding should exist")- (is (true? (:result finding)) "Analysis should be a success")- (is (= initial-analysis-time (:analyzed_at finding))- "Analysis should not have been re-run"))))))))+ (fn [{:keys [card-id]}]+ (jdbc/execute! one-off-dbs/*conn* ["CREATE TABLE \"ref_table\" (\"id\" INTEGER PRIMARY KEY, \"test_id\" INTEGER);"])+ (sync/sync-database! (mt/db))+ (let [ref-table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "ref_table")+ ref-field-id (t2/select-one-fn :id :model/Field :table_id ref-table-id :name "test_id")]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id ref-table-id})+ (record-processed! card-id)+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"ref_table\" ADD FOREIGN KEY (\"test_id\") REFERENCES \"test_table\"(\"id\");"])+ (sync/sync-database! (mt/db))+ (is (some? (t2/select-one-fn :fk_target_field_id :model/Field :id ref-field-id)))+ (is (pending? card-id)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.cljnew file mode 100644index 00000000000..a86b93136ab--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/analysis_finding_test.clj@@ -0,0 +1,52 @@+(ns metabase-enterprise.dependencies.task.analysis-finding-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [environ.core :as env]+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]+ [metabase-enterprise.dependencies.task.util :as task.util]+ [metabase.dependencies.hooks :as dependencies.hooks]+ [metabase.task.core :as task]+ [metabase.test :as mt])+ (:import+ (org.quartz JobDetail)))++(set! *warn-on-reflection* true)++(deftest ^:synchronized batch-size-zero-disables-job-test+ (testing "a batch size that isn't positive keeps the job from being scheduled by writes or at startup"+ (let [scheduled (atom [])+ scheduled-by (fn [thunk]+ (reset! scheduled [])+ (thunk)+ (count @scheduled))]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]+ (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "0")]+ (is (zero? (scheduled-by task.analysis-finding/trigger-analysis-finding-job!)))+ (is (zero? (scheduled-by #(task/init! ::task.analysis-finding/UpdateAnalysisFindings)))))+ (testing "a positive batch size schedules from the event trigger"+ (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "5")]+ (is (= 1 (scheduled-by task.analysis-finding/trigger-analysis-finding-job!)))))))))++(deftest ^:synchronized run-stops-without-feature-test+ (testing "a run without the dependencies feature schedules no next run, since the next write or token update triggers one"+ (let [scheduled (atom [])]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]+ (mt/with-premium-features #{}+ (task.util/run-and-reschedule! nil @#'task.analysis-finding/job))+ (is (empty? @scheduled))))))++(def ^:private job-key "metabase.dependencies.task.entity-check.job")++(defn- scheduled-job-names+ "Job names from captured [[task/schedule-task!]] calls, since a token update schedules every dependency job."+ [calls]+ (into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))++(deftest ^:synchronized token-update-restarts-job-test+ (testing "Setting a premium token must restart the analysis finding job, or enabling the feature on an idle instance+ leaves the periodic checker dead until the next process restart."+ (let [scheduled (atom [])]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]+ (mt/with-premium-features #{:dependencies}+ (dependencies.hooks/after-premium-token-update)+ (is (contains? (scheduled-job-names @scheduled) job-key)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljdeleted file mode 100644index 8f2ec778799..00000000000--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ /dev/null@@ -1,645 +0,0 @@-(ns metabase-enterprise.dependencies.task.backfill-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.backfill-test]}}}}}}- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [environ.core :as env]- [java-time.api :as t]- [metabase-enterprise.dependencies.calculation :as deps.calculation]- [metabase-enterprise.dependencies.models.dependency :as dependencies.model]- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]- [metabase-enterprise.dependencies.test-util :as deps.test]- [metabase.dependencies.hooks :as dependencies.hooks]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.premium-features.core :as premium-features]- [metabase.task.core :as task]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [toucan2.core :as t2])- (:import- (org.quartz JobKey)))--(set! *warn-on-reflection* true)--(use-fixtures :once (fixtures/initialize :db))--(defn- backfill-all-existing-entities!- []- (deps.test/synchronously-run-backfill!))--(defn- backfill-dependencies-single-trigger!- []- (mt/with-premium-features #{:dependencies}- (#'dependencies.backfill/backfill-dependencies!)))--(defn- mark-stale!- "Mark an entity as stale in dependency_status for testing."- [entity-type entity-id]- (deps.dependency-status/mark-stale! {entity-type #{entity-id}}))--(defn- assert-processed- "Assert that an entity has been processed (not stale, current version)."- [entity-type entity-id]- (is (t2/exists? :model/DependencyStatus- :entity_type entity-type- :entity_id entity-id- :stale false- :dependency_analysis_version dependencies.model/current-dependency-analysis-version)- (str "Expected " (name entity-type) " " entity-id " to be processed")))--(deftest ^:synchronized backfill-dependency-analysis-test- (testing "Test that the backfill job correctly processes stale entities"- (backfill-all-existing-entities!)- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]- (let [query (mt/mbql-query orders)]- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}- :model/Card {card2-id :id} {:dataset_query query}- :model/Card {card3-id :id} {:dataset_query query}]- ;; Mark all three cards as stale- (mark-stale! :card card1-id)- (mark-stale! :card card2-id)- (mark-stale! :card card3-id)- (let [stale-count (fn []- (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]- :stale true))]- (is (= 3 (stale-count)))- ;; first run, should process 2 cards- (is (true? (backfill-dependencies-single-trigger!)))- (is (= 1 (stale-count)))- ;; second run, should process the last card- (is (false? (backfill-dependencies-single-trigger!)))- (is (= 0 (stale-count)))- ;; third run, should not process anything- (is (false? (backfill-dependencies-single-trigger!)))- (is (= 0 (stale-count))))))))))--(deftest ^:synchronized backfill-transform-test- (testing "Test that transforms are correctly backfilled"- (backfill-all-existing-entities!)- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}- :model/Transform {transform-id :id} {:name "Test Transform"- :source {:type "query"- :query (mt/mbql-query nil- {:source-table (str "card__" card-id)})}- :target {:type "table" :name (mt/random-name)}}]- ;; Mark both as stale- (mark-stale! :card card-id)- (mark-stale! :transform transform-id)- (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))- (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (assert-processed :transform transform-id)- (is (t2/exists? :model/Dependency- :from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id (mt/id :orders)))- (is (t2/exists? :model/Dependency- :from_entity_type :transform :from_entity_id transform-id- :to_entity_type :card :to_entity_id card-id)))))))--(deftest ^:synchronized backfill-snippet-test- (testing "Test that cards with snippets are correctly backfilled"- (backfill-all-existing-entities!)- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]- (mt/with-premium-features #{}- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"- :content "SELECT 1"}- :model/Card {card-id :id} {:dataset_query {:database (mt/id)- :type :native- :native {:query "SELECT * FROM {{my_snippet}}"- :template-tags- {:my_snippet- {:id (str (random-uuid))- :name "my_snippet"- :display-name "my_snippet"- :type :snippet- :snippet-name "my_snippet"- :snippet-id snippet-id}}}}}]- ;; Mark both as stale- (mark-stale! :card card-id)- (mark-stale! :snippet snippet-id)- (is (false? (t2/exists? :model/Dependency- :from_entity_type :card :from_entity_id card-id- :to_entity_type :snippet :to_entity_id snippet-id)))- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (assert-processed :snippet snippet-id)- (is (t2/exists? :model/Dependency- :from_entity_type :card :from_entity_id card-id- :to_entity_type :snippet :to_entity_id snippet-id)))))))--(deftest ^:synchronized backfill-snippet-in-snippet-test- (testing "A snippet referencing another snippet produces a snippet->snippet dependency"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner_snip" :content "1 = 1"}- :model/NativeQuerySnippet {outer-id :id} {:name "outer_snip" :content "SELECT * WHERE {{snippet: inner_snip}}"}]- (mark-stale! :snippet inner-id)- (mark-stale! :snippet outer-id)- (is (false? (t2/exists? :model/Dependency- :from_entity_type :snippet :from_entity_id outer-id- :to_entity_type :snippet :to_entity_id inner-id)))- (backfill-dependencies-single-trigger!)- (assert-processed :snippet inner-id)- (assert-processed :snippet outer-id)- (is (t2/exists? :model/Dependency- :from_entity_type :snippet :from_entity_id outer-id- :to_entity_type :snippet :to_entity_id inner-id))))))--(deftest ^:synchronized backfill-snippet-referencing-card-test- (testing "A snippet referencing a card produces a snippet->card dependency"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}- :model/NativeQuerySnippet {snippet-id :id} {:name "card_ref_snip"- :content (format "SELECT * FROM {{#%d}}" card-id)}]- (mark-stale! :card card-id)- (mark-stale! :snippet snippet-id)- (is (false? (t2/exists? :model/Dependency- :from_entity_type :snippet :from_entity_id snippet-id- :to_entity_type :card :to_entity_id card-id)))- (backfill-dependencies-single-trigger!)- (assert-processed :snippet snippet-id)- (is (t2/exists? :model/Dependency- :from_entity_type :snippet :from_entity_id snippet-id- :to_entity_type :card :to_entity_id card-id))))))--(deftest ^:synchronized backfill-idempotency-test- (testing "Running the backfill multiple times should be idempotent"- (backfill-all-existing-entities!)- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id (mt/id :orders))))- ;; First run- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- ;; Second run - should not change anything- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id (mt/id :orders))))))))--(defn- wait-for-condition- [predicate timeout-ms]- (let [limit (+ (System/currentTimeMillis) timeout-ms)]- (loop []- (Thread/sleep 100)- (or (predicate)- (when (< (System/currentTimeMillis) limit)- (recur))))))--(deftest ^:synchronized backfill-scheduling-test- (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"- (backfill-all-existing-entities!)- (mt/with-temp-scheduler!- (with-redefs [env/env (assoc env/env- :mb-dependency-backfill-batch-size "1"- :mb-dependency-backfill-delay-minutes "0"- :mb-dependency-backfill-variance-minutes "0")]- (let [card-data {:dataset_query (mt/mbql-query orders)}]- (mt/test-helpers-set-global-values!- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card1-id :id} card-data- :model/Card {card2-id :id} card-data]- (mark-stale! :card card1-id)- (mark-stale! :card card2-id)- (let [processed? (fn []- (= 2 (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id]]- :stale false- :dependency_analysis_version dependencies.model/current-dependency-analysis-version)))]- (is (not (processed?)))- (mt/with-premium-features #{:dependencies}- (task/init! ::dependencies.backfill/DependencyBackfill)- (wait-for-condition processed? 2500)- (is (processed?))))))))))))--(deftest ^:synchronized backfill-no-trigger-pile-up-test- (testing "Scheduling new runs while the backfill job is executing keeps the trigger count ≤ 1"- (let [job-started (promise)- can-finish (promise)]- ;; the job body runs on a Quartz worker thread, which does not inherit *local-redefs* —- ;; only a root swap is visible there- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [dependencies.backfill/backfill-dependencies!- (fn []- (deliver job-started true)- (deref can-finish 1500 :timeout)- ;; non-nil → the job will hit the in-job self-reschedule path on exit- true)]- (mt/with-temp-scheduler!- ;; Start the job immediately.- (#'dependencies.backfill/schedule-run! (task/scheduler) 0)- (is (true? (deref job-started 1000 :timeout))- "BackfillDependencies job did not start within 1s")- (let [scheduler (task/scheduler)- job-key (JobKey. "metabase.task.dependency-backfill.job")- trigger-count #(count (.getTriggersOfJob scheduler job-key))]- ;; Spam 50 event-driven schedule attempts while the job is in-flight.- (dotimes [_ 50]- (dependencies.backfill/trigger-backfill-job!))- (is (<= (trigger-count) 1)- (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))- ;; Release the job; on exit it self-reschedules via the in-job path.- (deliver can-finish true)- (Thread/sleep 200)- (is (<= (trigger-count) 1)- (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))--(deftest ^:synchronized backfill-error-logging-test- (testing "When calculate-deps throws, the error is logged and the entity remains stale"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (with-redefs [env/env (assoc env/env- :mb-dependency-backfill-delay-minutes "0"- :mb-dependency-backfill-variance-minutes "0")- deps.calculation/calculate-deps- (fn [_ _] (throw (ex-info "Simulated error" {})))]- (backfill-dependencies-single-trigger!))- ;; Entity should still be stale — not processed, not lost- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; No dependencies should have been created- (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))- ;; Failure recorded in table- (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus- :entity_type :card :entity_id card-id)))))))--(deftest ^:synchronized backfill-partial-batch-failure-test- (testing "A failure on one entity doesn't prevent other entities from being processed"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}- :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]- (mark-stale! :card good-card-id)- (mark-stale! :card bad-card-id)- (let [original-calculate-deps deps.calculation/calculate-deps]- (with-redefs [env/env (assoc env/env- :mb-dependency-backfill-delay-minutes "0"- :mb-dependency-backfill-variance-minutes "0")- deps.calculation/calculate-deps- (fn [entity-type entity]- (if (= (:id entity) bad-card-id)- (throw (ex-info "Simulated error" {}))- (original-calculate-deps entity-type entity)))]- (backfill-dependencies-single-trigger!)))- ;; Good card should be processed successfully- (assert-processed :card good-card-id)- (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))- ;; Bad card should remain stale with failure recorded- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id bad-card-id :stale true))- (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus- :entity_type :card :entity_id bad-card-id)))))))--(deftest ^:synchronized backfill-terminal-failure-test- (testing "Entities should be marked as terminally broken after MAX_RETRIES failures"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- (let [compute-attempts (volatile! 0)- failures (inc @#'dependencies.backfill/max-retries)]- (with-redefs [env/env (assoc env/env- :mb-dependency-backfill-delay-minutes "0"- :mb-dependency-backfill-variance-minutes "0")- ;; Make compute-deps-for-entity! throw- deps.calculation/calculate-deps- (fn [_ _]- (vswap! compute-attempts inc)- (throw (ex-info "Simulated computation error" {:card-id card-id})))]- ;; fail MAX_RETRIES + 1 times- (while (< @compute-attempts failures)- (backfill-dependencies-single-trigger!))))- ;; verify card is still stale (not processed)- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; verify subsequent runs don't process it- (backfill-dependencies-single-trigger!)- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))))))--(deftest ^:synchronized backfill-delayed-retry-test- (testing "Failed entities should be retried after their delay period expires"- (backfill-all-existing-entities!)- (let [compute-attempts (volatile! 0)]- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (with-redefs [env/env (assoc env/env- :mb-dependency-backfill-delay-minutes "1")- deps.calculation/calculate-deps- (fn [_ _entity]- (if (zero? @compute-attempts)- (do- (vswap! compute-attempts inc)- (throw (ex-info "Simulated computation error" {:card-id card-id})))- ;; Return valid deps on subsequent attempts- {:table #{(mt/id :orders)}}))]- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; first failure - should be put into retry state- (while (zero? @compute-attempts)- (backfill-dependencies-single-trigger!))- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; advance time by less than retry delay - should NOT be processed- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))- (backfill-dependencies-single-trigger!))- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; advance time by more than retry delay - should be processed- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))- (backfill-dependencies-single-trigger!))- (assert-processed :card card-id)))))))--(deftest backfill-card-does-not-cause-revision-test- (testing "backfilling a card does not create a new revision or audit log entry"- (backfill-all-existing-entities!)- (mt/with-premium-features #{:dependencies :audit-app}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)- deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]- (is (= 0 revision-count-before))- (is (= 0 deps-before))- (backfill-dependencies-single-trigger!)- (let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)- deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]- (is (= 0 revision-count-after))- (assert-processed :card card-id)- (is (= 1 deps-after))))))))--(deftest ^:synchronized backfill-dependencies-on-token-update-test- (testing "Token update triggers the backfill job (entities remain stale until job runs)"- (backfill-all-existing-entities!)- (let [query (mt/mbql-query orders)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}- :model/Card {card2-id :id} {:dataset_query query}- :model/Card {card3-id :id} {:dataset_query query}]- ;; Mark all as stale- (mark-stale! :card card1-id)- (mark-stale! :card card2-id)- (mark-stale! :card card3-id)- ;; token update triggers the async job, not synchronous processing- (dependencies.hooks/after-premium-token-update)- ;; entities are still stale (job hasn't run yet in this test)- (is (= 3 (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]- :stale true)))- ;; manually run the backfill to verify it processes them- (backfill-all-existing-entities!)- (is (= 3 (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]- :stale false- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))--(deftest ^:synchronized backfill-version-outdated-test- (testing "Entities with outdated version in dependency_status get reprocessed"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- ;; Create a dependency_status entry with an old version (not stale, but outdated)- (t2/insert! :model/DependencyStatus {:entity_type :card- :entity_id card-id- :dependency_analysis_version 0- :stale false})- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id (mt/id :orders)))))))--(deftest ^:synchronized backfill-no-status-row-test- (testing "Entities with no dependency_status row yet get picked up and processed by the backfill"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- ;; Precondition: the temp card was inserted directly (no :event/card-create), so it has no- ;; status row at all. This is the case the left-join in instances-for-dependency-calculation- ;; must catch — guard it so the test can't silently pass without exercising that branch.- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))- "Expected the temp card to start with no dependency_status row")- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id (mt/id :orders)))))))--(deftest ^:synchronized has-stale-or-outdated?-counts-no-status-row-test- (testing "has-stale-or-outdated? (backing /backfill-status) stays consistent with what the backfill processes"- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- ;; Clean slate: nothing left to process.- (is (false? (deps.dependency-status/has-stale-or-outdated?))- "Expected no pending work after backfilling all existing entities")- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- ;; A card with no status row is pending work, even though no DependencyStatus row exists.- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id)))- (is (true? (deps.dependency-status/has-stale-or-outdated?))- "Expected pending work: a card with no status row still needs calculation")- (backfill-dependencies-single-trigger!)- (assert-processed :card card-id)- (is (false? (deps.dependency-status/has-stale-or-outdated?))- "Expected no pending work once the card has been processed")))))--(deftest ^:synchronized batch-size-zero-suppresses-event-triggers-but-stays-warm-test- (testing "A non-positive batch size stops the job doing work, but leaves it on its slow periodic schedule so it- resumes if the setting ever becomes positive. What it must not do is keep firing the 1-second event-driven- trigger, which is what made a disabled job wake roughly once a second on a busy instance -- entity changes- fire it, and it never consulted the batch size."- (let [scheduled (atom [])- scheduled-by (fn [thunk]- (reset! scheduled [])- (thunk)- (count @scheduled))]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]- (when (some #(str/includes? (str %) "dependency-backfill") args)- (swap! scheduled conj args))- nil)]- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")]- (testing "the 1-second event-driven trigger is suppressed"- (is (zero? (scheduled-by dependencies.backfill/trigger-backfill-job!))))- (testing "but task/init! still puts the job on its periodic schedule"- (is (= 1 (scheduled-by #(task/init! ::dependencies.backfill/DependencyBackfill)))))- (testing "and a run keeps the chain going rather than stopping"- (mt/with-premium-features #{:dependencies}- (is (= 1 (scheduled-by #(#'dependencies.backfill/run-and-reschedule! nil)))))))- (testing "a positive batch size schedules from the event trigger too"- (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "5")]- (is (= 1 (scheduled-by dependencies.backfill/trigger-backfill-job!)))))))))--(deftest ^:synchronized backfill-batch-shares-one-metadata-provider-test- (testing "GHY-4251: every entity in a batch must share one MetadataProvider per database. Reading an entity attaches- a provider to its query, so without a batch-scoped cache each entity gets a private one; native dep- calculation then fills each with that database's table metadata and the batch retains one copy per entity,- which is what exhausted the heap on instances with large warehouses."- (backfill-all-existing-entities!)- (let [providers (atom [])- calculate-deps (mt/original-fn #'deps.calculation/calculate-deps)- native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}- :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]- (mark-stale! :card card1-id)- (mark-stale! :card card2-id)- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps- (fn [entity-type entity]- (when-let [mp (:lib/metadata (:dataset_query entity))]- (swap! providers conj mp))- (calculate-deps entity-type entity))]- (backfill-dependencies-single-trigger!))- (testing "both cards were analyzed, so the assertion below is meaningful"- (is (<= 2 (count @providers))))- (testing "and they were analyzed through the same provider instance"- ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be- ;; satisfied by two separate caches and would not detect the regression.- (is (= 1 (count (into #{} (map #(System/identityHashCode %)) @providers))))))))))--(deftest ^:synchronized backfill-records-failure-for-fatal-error-test- (testing "GHY-4251: a fatal Error must record the entity's failure before propagating. Only Exception was caught,- so an OutOfMemoryError escaped without recording anything and the next run selected the identical batch --- a crash loop that never made progress. Recording first means the entity backs off and eventually goes- terminal, so the batch drains even if one entity always kills the process."- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (Error. "simulated fatal error")))]- (testing "the Error propagates instead of being swallowed, failing the job"- (is (thrown-with-msg? Error #"simulated fatal error"- (backfill-dependencies-single-trigger!)))))- (testing "and the failure was recorded first, so the next run backs off instead of replaying the batch"- (let [{:keys [fail_count next_retry_at terminal]}- (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)]- (is (= 1 fail_count))- (is (some? next_retry_at))- (is (false? terminal))))))))--(deftest ^:synchronized pending-retries-licence-states-test- (testing "A pending retry keeps the backfill job scheduled, but only when the licence permits acting on it. A retry- marker is cleared by processing the entity -- on success via upsert-status!, or by going terminal past- max-retries -- and both are gated on the feature, so without it the job could never resolve the condition- keeping it awake and rescheduled hourly forever.-- The licence is resolved with canonically-has-feature? rather than has-feature?, which collapses 'no licence'- and 'could not check' into the same false. Both inputs to the job's reschedule decision consult the licence,- so a transient token failure would take the job down permanently -- nothing re-fires when the check- recovers, because the token never changed."- (backfill-all-existing-entities!)- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (mark-stale! :card card-id)- (backfill-dependencies-single-trigger!)- (deps.dependency-status/record-failure! :card card-id 5 60)- (is (t2/exists? :model/DependencyStatus- :entity_type :card :entity_id card-id- :terminal false :next_retry_at [:not= nil])- "test setup: the card must be in retry backoff")- (let [actionable? (fn [licence]- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly licence)]- (#'dependencies.backfill/has-pending-retries?)))]- (testing "licensed: the pending retry keeps the job scheduled"- (is (true? (actionable? true))))- (testing "indeterminate: treated as actionable, so a blip cannot end the chain"- (is (true? (actionable? nil))))- (testing "definitively unlicensed: not actionable"- (is (false? (actionable? false))))))))--(deftest ^:synchronized backfill-records-failure-for-never-processed-entity-test- (testing "An entity with no dependency_status row yet must still get a failure recorded when it fails. Such entities- are explicitly selected for processing (instances-for-dependency-calculation matches a null status row), so- on a first backfill every entity is in this state -- exactly the population most likely to fail on a large- instance. record-failure! only updated an existing row, so nothing was recorded and the next run selected- the identical batch: the crash loop, unbroken for the entities most likely to cause it."- (backfill-all-existing-entities!)- (mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))- "test setup: the card must have no status row, which is what makes it eligible")- (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (ex-info "boom" {})))]- (backfill-dependencies-single-trigger!))- (testing "the failure is recorded, so the entity backs off instead of being reselected unchanged"- (let [{:keys [fail_count next_retry_at]}- (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)]- (is (= 1 fail_count))- (is (some? next_retry_at))))))))--(deftest ^:synchronized fatal-error-still-reschedules-test- (testing "A failure must not leave the job unscheduled. The job is one-shot self-rescheduling with no cron- backstop, so an Error propagating out of the batch skipped schedule-run! entirely and the chain ended- until a content-change event or a restart. A real OutOfMemoryError usually takes the process with it and- task/init! recovers on boot, but survivable Errors -- StackOverflowError from deeply nested SQL, an- AssertionError, a LinkageError -- leave a live process with a dead job."- (let [scheduled (atom [])]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)- dependencies.backfill/backfill-dependencies!- (fn [& _] (throw (Error. "simulated fatal error")))]- (testing "the Error still propagates"- (is (thrown-with-msg? Error #"simulated fatal error"- (#'dependencies.backfill/run-and-reschedule! nil))))- (testing "and the next run was scheduled anyway"- (is (= 1 (count @scheduled))))))))--(defn- table-dependency-ids- "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."- [entity-type entity-id]- (t2/select-fn-set :to_entity_id :model/Dependency- :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))--(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test- (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,- recomputes the cards whose SQL expands it"- (backfill-all-existing-entities!)- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}- :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}- :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)- "{{snippet: birds outer}}")}]- (backfill-all-existing-entities!)- (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))- (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})- (backfill-all-existing-entities!)- (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))--(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test- (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"- (backfill-all-existing-entities!)- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)]- (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}- :model/Card {card-id :id} {:dataset_query- (lib/native-query- mp- (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"- orders-id))}]- (backfill-all-existing-entities!)- (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))- (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id- :to_entity_type :card :to_entity_id orders-id)))))))--(deftest ^:synchronized backfill-records-table-sync-finds-later-test- (testing "a table a native query names before sync finds it gets its dependency once a sync of the database finds it"- (backfill-all-existing-entities!)- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)- "SELECT * FROM PUBLIC.BIRDS_LATE")}- :model/Card {known-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)- "SELECT * FROM VENUES")}]- (backfill-all-existing-entities!)- (is (empty? (table-dependency-ids :card card-id)))- (is (true? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id card-id)))- (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id known-id)))- (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]- (dependencies.hooks/after-database-sync (mt/id))- (backfill-all-existing-entities!)- (is (= #{late-id} (table-dependency-ids :card card-id)))- (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus- :entity_type :card :entity_id card-id))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.cljsimilarity index 52%rename from enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljrename to enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.cljindex b07825ca508..d190b2488c1 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj@@ -1,43 +1,549 @@-(ns metabase-enterprise.dependencies.events-test+(ns metabase-enterprise.dependencies.task.dependency-test+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.dependency-test]}}}}}}(:require- [clojure.test :refer [deftest is testing]]- [metabase-enterprise.dependencies.findings :as deps.findings]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [environ.core :as env]+ [metabase-enterprise.dependencies.calculation :as deps.calculation]+ [metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.hooks :as deps.hooks]- [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency :as dependencies.model][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]+ [metabase-enterprise.dependencies.settings :as deps.settings]+ [metabase-enterprise.dependencies.task.dependency :as task.dependency]+ [metabase-enterprise.dependencies.task.util :as task.util][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.serialization.test-util :as ts][metabase-enterprise.serialization.v2.load :as serdes.load]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.serialization :as serdes]+ [metabase.task.core :as task][metabase.test :as mt]- [toucan2.core :as t2]))+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2])+ (:import+ (org.quartz JobKey)))-(defn- run-with-dependencies-setup! [thunk]- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding :model/DependencyStatus]- (lib-be/with-metadata-provider-cache- (let [mp (mt/metadata-provider)]- (thunk mp)))))))+(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(defn- backfill-all-existing-entities!+ []+ (deps.test/synchronously-run-dependency-job!))++(defn- backfill-dependencies-single-trigger!+ []+ (mt/with-premium-features #{:dependencies}+ (#'task.dependency/update-dependencies!)))++(defn- forget-status!+ "Delete the dependency status of an entity, so the dependency job computes its dependencies again."+ [entity-type entity-id]+ (dependencies.db/delete-dependency-statuses! entity-type [entity-id]))-(defn- assert-stale- "Assert that the given entity is marked stale in dependency_status."+(def ^:private pending? deps.test/dependency-pending?)++(defn- processed?+ "Whether the dependency job computed the dependencies of the entity `entity-type` `entity-id` with the current logic+ and nothing marked them stale since."[entity-type entity-id]- (is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)- (str "Expected " (name entity-type) " " entity-id " to be marked stale")))+ (and (t2/exists? :model/DependencyStatus+ :entity_type entity-type+ :entity_id entity-id+ :analysis_version dependencies.model/current-dependency-analysis-version)+ (not (pending? entity-type entity-id))))(defn- assert-pending- "Assert that the given entity awaits the backfill job, having no dependency_status row or a stale one."+ "Assert that the dependency job is to compute the dependencies of the entity `entity-type` `entity-id`."[entity-type entity-id]- (is (not (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale false))+ (is (pending? entity-type entity-id)(str "Expected " (name entity-type) " " entity-id " to be pending")))+(defn- assert-processed+ "Assert that an entity has been processed."+ [entity-type entity-id]+ (is (processed? entity-type entity-id)+ (str "Expected " (name entity-type) " " entity-id " to be processed")))++(deftest ^:synchronized backfill-dependency-analysis-test+ (testing "the dependency job processes the pending entities in batches, asking to run again while a batch took any"+ (backfill-all-existing-entities!)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (let [query (mt/mbql-query orders)]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}+ :model/Card {card2-id :id} {:dataset_query query}+ :model/Card {card3-id :id} {:dataset_query query}]+ (forget-status! :card card1-id)+ (forget-status! :card card2-id)+ (forget-status! :card card3-id)+ (let [pending-count (fn [] (count (filter #(pending? :card %) [card1-id card2-id card3-id])))]+ (is (= 3 (pending-count)))+ ;; first run, should process 2 cards+ (is (true? (backfill-dependencies-single-trigger!)))+ (is (= 1 (pending-count)))+ ;; second run, should process the last card+ (is (true? (backfill-dependencies-single-trigger!)))+ (is (= 0 (pending-count)))+ ;; third run, should not process anything+ (is (nil? (backfill-dependencies-single-trigger!)))+ (is (= 0 (pending-count))))))))))++(deftest ^:synchronized backfill-transform-test+ (testing "Test that transforms are correctly backfilled"+ (backfill-all-existing-entities!)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}+ :model/Transform {transform-id :id} {:name "Test Transform"+ :source {:type "query"+ :query (mt/mbql-query nil+ {:source-table (str "card__" card-id)})}+ :target {:type "table" :name (mt/random-name)}}]+ (forget-status! :card card-id)+ (forget-status! :transform transform-id)+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))+ (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (assert-processed :transform transform-id)+ (is (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders)))+ (is (t2/exists? :model/Dependency+ :from_entity_type :transform :from_entity_id transform-id+ :to_entity_type :card :to_entity_id card-id)))))))++(deftest ^:synchronized backfill-snippet-test+ (testing "Test that cards with snippets are correctly backfilled"+ (backfill-all-existing-entities!)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"+ :content "SELECT 1"}+ :model/Card {card-id :id} {:dataset_query {:database (mt/id)+ :type :native+ :native {:query "SELECT * FROM {{my_snippet}}"+ :template-tags+ {:my_snippet+ {:id (str (random-uuid))+ :name "my_snippet"+ :display-name "my_snippet"+ :type :snippet+ :snippet-name "my_snippet"+ :snippet-id snippet-id}}}}}]+ (forget-status! :card card-id)+ (forget-status! :snippet snippet-id)+ (is (false? (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :snippet :to_entity_id snippet-id)))+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (assert-processed :snippet snippet-id)+ (is (t2/exists? :model/Dependency+ :from_entity_type :card :from_entity_id card-id+ :to_entity_type :snippet :to_entity_id snippet-id)))))))++(deftest ^:synchronized backfill-snippet-in-snippet-test+ (testing "A snippet referencing another snippet produces a snippet->snippet dependency"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "inner_snip" :content "1 = 1"}+ :model/NativeQuerySnippet {outer-id :id} {:name "outer_snip" :content "SELECT * WHERE {{snippet: inner_snip}}"}]+ (forget-status! :snippet inner-id)+ (forget-status! :snippet outer-id)+ (is (false? (t2/exists? :model/Dependency+ :from_entity_type :snippet :from_entity_id outer-id+ :to_entity_type :snippet :to_entity_id inner-id)))+ (backfill-dependencies-single-trigger!)+ (assert-processed :snippet inner-id)+ (assert-processed :snippet outer-id)+ (is (t2/exists? :model/Dependency+ :from_entity_type :snippet :from_entity_id outer-id+ :to_entity_type :snippet :to_entity_id inner-id))))))++(deftest ^:synchronized backfill-snippet-referencing-card-test+ (testing "A snippet referencing a card produces a snippet->card dependency"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}+ :model/NativeQuerySnippet {snippet-id :id} {:name "card_ref_snip"+ :content (format "SELECT * FROM {{#%d}}" card-id)}]+ (forget-status! :card card-id)+ (forget-status! :snippet snippet-id)+ (is (false? (t2/exists? :model/Dependency+ :from_entity_type :snippet :from_entity_id snippet-id+ :to_entity_type :card :to_entity_id card-id)))+ (backfill-dependencies-single-trigger!)+ (assert-processed :snippet snippet-id)+ (is (t2/exists? :model/Dependency+ :from_entity_type :snippet :from_entity_id snippet-id+ :to_entity_type :card :to_entity_id card-id))))))++(deftest ^:synchronized backfill-idempotency-test+ (testing "Running the backfill multiple times should be idempotent"+ (backfill-all-existing-entities!)+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (forget-status! :card card-id)+ (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders))))+ ;; First run+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ ;; Second run - should not change anything+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders))))))))++(defn- wait-for-condition+ [predicate timeout-ms]+ (let [limit (+ (System/currentTimeMillis) timeout-ms)]+ (loop []+ (Thread/sleep 100)+ (or (predicate)+ (when (< (System/currentTimeMillis) limit)+ (recur))))))++(deftest ^:synchronized backfill-scheduling-test+ (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"+ (backfill-all-existing-entities!)+ (mt/with-temp-scheduler!+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-batch-size "1"+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")]+ (let [card-data {:dataset_query (mt/mbql-query orders)}]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card1-id :id} card-data+ :model/Card {card2-id :id} card-data]+ (forget-status! :card card1-id)+ (forget-status! :card card2-id)+ (let [both-processed? #(every? (partial processed? :card) [card1-id card2-id])]+ (is (not (both-processed?)))+ (mt/with-premium-features #{:dependencies}+ (task/init! ::task.dependency/UpdateDependencies)+ (wait-for-condition both-processed? 2500)+ (is (both-processed?))))))))))))++(deftest ^:synchronized backfill-no-trigger-pile-up-test+ (testing "Scheduling new runs while the dependency job is executing keeps the trigger count ≤ 1"+ (let [job-started (promise)+ can-finish (promise)]+ ;; the job body runs on a Quartz worker thread, which does not inherit *local-redefs* —+ ;; only a root swap is visible there+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [task.dependency/update-dependencies!+ (fn []+ (deliver job-started true)+ (deref can-finish 1500 :timeout)+ ;; non-nil → the job will hit the in-job self-reschedule path on exit+ true)]+ (mt/with-temp-scheduler!+ ;; Start the job immediately.+ (task.util/schedule-run! (task/scheduler) @#'task.dependency/job 0)+ (is (true? (deref job-started 1000 :timeout))+ "UpdateDependencies job did not start within 1s")+ (let [scheduler (task/scheduler)+ job-key (JobKey. "metabase.task.dependency-backfill.job")+ trigger-count #(count (.getTriggersOfJob scheduler job-key))]+ ;; Spam 50 event-driven schedule attempts while the job is in-flight.+ (dotimes [_ 50]+ (task.dependency/trigger-dependency-job!))+ (is (<= (trigger-count) 1)+ (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))+ ;; Release the job; on exit it self-reschedules via the in-job path.+ (deliver can-finish true)+ (Thread/sleep 200)+ (is (<= (trigger-count) 1)+ (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))++(deftest ^:synchronized backfill-error-logging-test+ (testing "When calculate-deps throws, the error is logged and the entity is computed again on the next run"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (forget-status! :card card-id)+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")+ deps.calculation/calculate-deps+ (fn [_ _] (throw (ex-info "Simulated error" {})))]+ (backfill-dependencies-single-trigger!))+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))+ (is (pending? :card card-id))))))++(deftest ^:synchronized backfill-partial-batch-failure-test+ (testing "A failure on one entity doesn't prevent other entities from being processed"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}+ :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]+ (forget-status! :card good-card-id)+ (forget-status! :card bad-card-id)+ (let [original-calculate-deps deps.calculation/calculate-deps]+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")+ deps.calculation/calculate-deps+ (fn [entity-type entity]+ (if (= (:id entity) bad-card-id)+ (throw (ex-info "Simulated error" {}))+ (original-calculate-deps entity-type entity)))]+ (backfill-dependencies-single-trigger!)))+ ;; Good card should be processed successfully+ (assert-processed :card good-card-id)+ (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))+ (is (pending? :card bad-card-id))))))++(deftest ^:synchronized backfill-failure-waits-for-change-test+ (testing "An entity whose calculation keeps failing is given up on after dependency-max-failures runs, until it changes"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (forget-status! :card card-id)+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps+ (fn [_ _] (throw (ex-info "Simulated computation error" {:card-id card-id})))]+ (dotimes [_ (deps.settings/dependency-max-failures)]+ (backfill-dependencies-single-trigger!)))+ (is (not (pending? :card card-id)))+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))+ (forget-status! :card card-id)+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)))))++(deftest backfill-card-does-not-cause-revision-test+ (testing "backfilling a card does not create a new revision or audit log entry"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies :audit-app}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (forget-status! :card card-id)+ (let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)+ deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]+ (is (= 0 revision-count-before))+ (is (= 0 deps-before))+ (backfill-dependencies-single-trigger!)+ (let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)+ deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]+ (is (= 0 revision-count-after))+ (assert-processed :card card-id)+ (is (= 1 deps-after))))))))++(deftest ^:synchronized backfill-dependencies-on-token-update-test+ (testing "Token update triggers the dependency job (entities remain pending until job runs)"+ (backfill-all-existing-entities!)+ (let [query (mt/mbql-query orders)]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}+ :model/Card {card2-id :id} {:dataset_query query}+ :model/Card {card3-id :id} {:dataset_query query}]+ (forget-status! :card card1-id)+ (forget-status! :card card2-id)+ (forget-status! :card card3-id)+ ;; token update triggers the async job, not synchronous processing+ (dependencies.hooks/after-premium-token-update)+ (is (every? #(pending? :card %) [card1-id card2-id card3-id]))+ ;; manually run the backfill to verify it processes them+ (backfill-all-existing-entities!)+ (is (every? #(processed? :card %) [card1-id card2-id card3-id])))))))++(deftest ^:synchronized backfill-version-outdated-test+ (testing "Entities with outdated version in dependency_status get reprocessed"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (backfill-dependencies-single-trigger!)+ (t2/update! :model/DependencyStatus :entity_type :card :entity_id card-id {:analysis_version 0})+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))++(deftest ^:synchronized backfill-no-status-row-test+ (testing "Entities with no dependency_status row yet get picked up and processed by the backfill"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))+ "Expected the temp card to start with no dependency_status row")+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))++(deftest ^:synchronized has-pending-entities?-counts-no-status-row-test+ (testing "has-pending-entities? (backing /backfill-status) stays consistent with what the dependency job processes"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ ;; Clean slate: nothing left to process.+ (is (false? (deps.dependency-status/has-pending-entities?))+ "Expected no pending work after backfilling all existing entities")+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ ;; A card with no status row is pending work, even though no DependencyStatus row exists.+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id)))+ (is (true? (deps.dependency-status/has-pending-entities?))+ "Expected pending work: a card with no status row still needs calculation")+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (is (false? (deps.dependency-status/has-pending-entities?))+ "Expected no pending work once the card has been processed")))))++(deftest ^:synchronized batch-size-zero-disables-job-test+ (testing "a batch size that isn't positive keeps the job from being scheduled by writes or at startup"+ (let [scheduled (atom [])+ scheduled-by (fn [thunk]+ (reset! scheduled [])+ (thunk)+ (count @scheduled))]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]+ (when (some #(str/includes? (str %) "dependency-backfill") args)+ (swap! scheduled conj args))+ nil)]+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")]+ (is (zero? (scheduled-by task.dependency/trigger-dependency-job!)))+ (is (zero? (scheduled-by #(task/init! ::task.dependency/UpdateDependencies)))))+ (testing "a positive batch size schedules from the event trigger"+ (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "5")]+ (is (= 1 (scheduled-by task.dependency/trigger-dependency-job!)))))))))++(deftest ^:synchronized backfill-batch-shares-one-metadata-provider-test+ (testing "GHY-4251: every entity in a batch must share one MetadataProvider per database. Reading an entity attaches+ a provider to its query, so without a batch-scoped cache each entity gets a private one; native dep+ calculation then fills each with that database's table metadata and the batch retains one copy per entity,+ which is what exhausted the heap on instances with large warehouses."+ (backfill-all-existing-entities!)+ (let [providers (atom [])+ calculate-deps (mt/original-fn #'deps.calculation/calculate-deps)+ native-card (fn [sql] {:database (mt/id) :type :native :native {:query sql}})]+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query (native-card "select id from orders")}+ :model/Card {card2-id :id} {:dataset_query (native-card "select id from products")}]+ (forget-status! :card card1-id)+ (forget-status! :card card2-id)+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps+ (fn [entity-type entity]+ (when-let [mp (:lib/metadata (:dataset_query entity))]+ (swap! providers conj mp))+ (calculate-deps entity-type entity))]+ (backfill-dependencies-single-trigger!))+ (testing "both cards were analyzed, so the assertion below is meaningful"+ (is (<= 2 (count @providers))))+ (testing "and they were analyzed through the same provider instance"+ ;; Identity, not equality: providers compare equal when they wrap the same database id, so `=` would be+ ;; satisfied by two separate caches and would not detect the regression.+ (is (= 1 (count (into #{} (map #(System/identityHashCode %)) @providers))))))))))++(deftest ^:synchronized fatal-error-gives-up-after-max-failures-test+ (testing "GHY-4251: a fatal Error propagates and counts as a failure, so an entity that keeps causing one is given up+ on after dependency-max-failures runs instead of replaying the batch in a crash loop"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (forget-status! :card card-id)+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (Error. "simulated fatal error")))]+ (dotimes [run (deps.settings/dependency-max-failures)]+ (testing (str "run " (inc run) ": the Error propagates instead of being swallowed")+ (is (thrown-with-msg? Error #"simulated fatal error" (backfill-dependencies-single-trigger!))))+ (testing (str "run " (inc run) ": the entity stays pending until the last allowed failure")+ (is (= (< (inc run) (deps.settings/dependency-max-failures)) (pending? :card card-id))))))+ (testing "a write gives the entity fresh attempts"+ (dependencies.db/mark-dependency-statuses-stale! {:card #{card-id}})+ (is (pending? :card card-id)))))))++(deftest ^:synchronized failing-entity-gives-up-after-max-failures-test+ (testing "an entity with no dependency_status row yet whose computation keeps failing is retried until+ dependency-max-failures, then given up on until it changes"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id))+ "test setup: the card must have no status row, which is what makes it eligible")+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (ex-info "boom" {})))]+ (backfill-dependencies-single-trigger!)+ (is (pending? :card card-id))+ (dotimes [_ (dec (deps.settings/dependency-max-failures))]+ (backfill-dependencies-single-trigger!)))+ (is (not (pending? :card card-id)))+ (is (=? {:terminal true :fail_count (deps.settings/dependency-max-failures)}+ (t2/select-one :model/DependencyStatus :entity_type :card :entity_id card-id)))))))++(deftest ^:synchronized fatal-error-still-reschedules-test+ (testing "A failure must not leave the job unscheduled. The job is one-shot self-rescheduling with no cron+ backstop, so an Error propagating out of the batch skipped schedule-run! entirely and the chain ended+ until a content-change event or a restart. A real OutOfMemoryError usually takes the process with it and+ task/init! recovers on boot, but survivable Errors -- StackOverflowError from deeply nested SQL, an+ AssertionError, a LinkageError -- leave a live process with a dead job."+ (let [scheduled (atom [])]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)+ task.dependency/update-dependencies!+ (fn [& _] (throw (Error. "simulated fatal error")))]+ (testing "the Error still propagates"+ (is (thrown-with-msg? Error #"simulated fatal error"+ (task.util/run-and-reschedule! nil @#'task.dependency/job))))+ (testing "and the next run was scheduled anyway"+ (is (= 1 (count @scheduled))))))))++(defn- table-dependency-ids+ "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."+ [entity-type entity-id]+ (t2/select-fn-set :to_entity_id :model/Dependency+ :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))++(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test+ (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,+ recomputes the cards whose SQL expands it"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}+ :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "{{snippet: birds outer}}")}]+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))+ (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))++(deftest ^:synchronized snippet-new-upstream-marks-analysis-test+ (testing "storing a new upstream of a snippet marks the analysis of what expands it, which an edit of that upstream+ before the edge existed never marked"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet _ {:name "window inner" :content "VENUES"}+ :model/NativeQuerySnippet {outer-id :id} {:name "window outer" :content "SELECT * FROM VENUES"}+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "{{snippet: window outer}}")}]+ (backfill-all-existing-entities!)+ (t2/update! :model/NativeQuerySnippet outer-id {:content "SELECT * FROM {{snippet: window inner}}"})+ (deps.test/record-analysis-finding-processed! :card card-id)+ (backfill-dependencies-single-trigger!)+ (is (deps.test/analysis-finding-pending? :card card-id))))))++(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test+ (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}+ :model/Card {card-id :id} {:dataset_query+ (lib/native-query+ mp+ (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"+ orders-id))}]+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :card :to_entity_id orders-id)))))))+(deftest dashboard-update-sets-correct-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [products-id (mt/id :products)orders-id (mt/id :orders)@@ -59,7 +565,7 @@:card_id series-card-id:position 0}](assert-pending :dashboard dashboard-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :dashboard:from_entity_id dashboard-id:to_entity_type :card@@ -114,7 +620,7 @@:visualization_settings {:column_settings {"[\"name\",\"CATEGORY\"]" {:click_behavior {:type "link":linkType "dashboard":targetId column-click-target-dashboard-id}}}}}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :dashboard:from_entity_id dashboard-id:to_entity_type :card@@ -150,11 +656,11 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))(t2/delete! :model/Dashboard dashboard-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))(deftest document-update-sets-correct-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [products-id (mt/id :products)products (lib.metadata/table mp products-id)]@@ -171,7 +677,7 @@{:type "cardEmbed":attrs {"id" embedded-card-id}}]}}](assert-pending :document document-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :document:from_entity_id document-id:to_entity_type :card@@ -191,7 +697,7 @@:attrs {"entityId" products-id"model" "table"}}]}]})](t2/update! :model/Document document-id updated-doc)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :document:from_entity_id document-id:to_entity_type :dashboard@@ -203,12 +709,12 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))(t2/delete! :model/Document document-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))))))))(deftest sandbox-update-sets-correct-dependencies(mt/with-premium-features #{:sandboxes}- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "sandbox group"}:model/Card {card1-id :id} {}@@ -217,7 +723,7 @@:table_id (mt/id :products):card_id card1-id}](assert-pending :sandbox sandbox-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :sandbox:from_entity_id sandbox-id:to_entity_type :card@@ -225,7 +731,7 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? #{{:from_entity_type :sandbox:from_entity_id sandbox-id:to_entity_type :card@@ -233,30 +739,27 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))(t2/delete! :model/Sandbox sandbox-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))(deftest card-marks-pending-on-create-and-stale-on-update-test- (testing "A created card is pending for the backfill job, and changing its query marks it stale in dependency_status"- (run-with-dependencies-setup!+ (testing "a created card is pending for the dependency job, and any update makes it pending again"+ (deps.test/run-with-dependencies-setup!(fn [mp](let [products (lib.metadata/table mp (mt/id :products))orders (lib.metadata/table mp (mt/id :orders))](mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}](assert-pending :card card-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :products)))- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false))- (testing "a change to a column the dependencies do not come from leaves it up to date"- (t2/update! :model/Card card-id {:name "renamed"})- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false)))+ (is (not (pending? :card card-id)))(t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)})- (assert-stale :card card-id)))))))+ (assert-pending :card card-id)))))))(deftest metric-dimensions-update-marks-stale-and-includes-mapped-table-test(testing "a metric dimension write marks the metric card stale and recomputes deps, including mapped-dimension tables"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [base (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))(lib/aggregate (lib/count)))]@@ -270,8 +773,8 @@:dimension-id "550e8400-e29b-41d4-a716-446655440000":table-id (mt/id :categories):target [:field {} (mt/id :categories :name)]}]})- (assert-stale :card card-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :card card-id)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :categories))"the mapped dimension's table becomes a dependency")@@ -280,41 +783,26 @@"the query's own table remains a dependency")))))))(deftest snippet-marks-pending-on-create-and-stale-on-update-test- (testing "A created snippet is pending for the backfill job, and changing its content marks it stale in dependency_status"- (run-with-dependencies-setup!+ (testing "a created snippet is pending for the dependency job, and changing its content makes it pending again"+ (deps.test/run-with-dependencies-setup!(fn [_](mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "test snippet":content "SELECT 1"}](assert-pending :snippet snippet-id)- (deps.test/synchronously-run-backfill!)- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id snippet-id :stale false))+ (deps.test/synchronously-run-dependency-job!)+ (is (not (pending? :snippet snippet-id)))(t2/update! :model/NativeQuerySnippet snippet-id {:content "SELECT 2"})- (assert-stale :snippet snippet-id))))))--(deftest mark-stale-failure-fails-the-write-test- (testing "When mark-stale! throws, the update fails and the card keeps its query"- (run-with-dependencies-setup!- (fn [mp]- (let [products (lib.metadata/table mp (mt/id :products))- orders (lib.metadata/table mp (mt/id :orders))]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]- (deps.test/synchronously-run-backfill!)- (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!- (fn [_] (throw (ex-info "Simulated DB failure" {})))]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated DB failure"- (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)}))))- (is (=? {:stages [{:source-table (mt/id :products)}]}- (t2/select-one-fn :dataset_query :model/Card card-id)))))))))+ (assert-pending :snippet snippet-id))))))(deftest ^:synchronized native-transform-create-sets-dependencies-test(testing "a created native transform is picked up by the dependency calculation"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders"):type :query}](mt/with-temp [:model/Transform {transform-id :id} {:source source}](assert-pending :transform transform-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,@@ -323,13 +811,13 @@(deftest ^:synchronized mbql-transform-create-sets-dependencies-test(testing "a created mbql transform is picked up by the dependency calculation"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders))):type :query}](mt/with-temp [:model/Transform {transform-id :id} {:source source}](assert-pending :transform transform-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,@@ -338,7 +826,7 @@(deftest ^:synchronized python-transform-create-sets-dependencies-test(testing "a created python transform is picked up by the dependency calculation"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](let [source {:type :python,:source-database (mt/id),@@ -351,61 +839,16 @@"import pandas as pd\n\ndef transform(orders):\n return orders"}](mt/with-temp [:model/Transform {transform-id :id} {:source source}](assert-pending :transform transform-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,:to_entity_id (mt/id :orders)}](t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))-(deftest ^:synchronized database-delete-marks-source-transforms-stale-test- (testing "deleting a database marks its source transforms' analysis findings stale so they surface on /dependency-diagnostics/broken (GDGT-2447)"- (run-with-dependencies-setup!- (fn [_mp]- (mt/with-temp [:model/Database {db-id :id} {}- :model/Transform {transform-id :id}- {:source {:type :query- :query {:database db-id- :type :native- :native {:query "SELECT 1"}}}- :target {:schema "public" :name "out" :type :table}}- :model/AnalysisFinding _ {:analyzed_entity_type :transform- :analyzed_entity_id transform-id- :result true- :stale false- :analysis_version models.analysis-finding/*current-analysis-finding-version*- :analyzed_at :%now}]- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))- "baseline: existing finding is fresh")- (t2/delete! :model/Database db-id)- (is (nil? (t2/select-one :model/Database db-id))- "the Database row is actually gone")- (assert-stale :transform transform-id)- (deps.test/synchronously-run-backfill!)- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))- "after db-delete the backfill marks the transform's finding stale")- (testing "and re-running the entity-check job flips :result false so the transform shows on broken-diagnostics"- (#'task.entity-check/check-entities!)- (let [{:keys [stale result]} (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id)]- (is (false? stale))- (is (false? result)))- (testing "an analysis_finding_error row points the transform at itself so the diagnostics page (`/graph/breaking`) surfaces it"- (is (=? {:source_entity_type :transform- :source_entity_id transform-id- :error_type :validation-exception-error}- (t2/select-one :model/AnalysisFindingError- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))))))))))-(deftest ^:synchronized transform-run-updates-dependencies-test(testing "transform run events trigger dependency calculations"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}@@ -419,7 +862,7 @@(deftest ^:synchronized python-transform-update-handles-downstream-dependencies-test(testing "python transform updates handle downstream dependencies"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](let [source {:type :python,:source-database (mt/id),@@ -444,8 +887,8 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "keeping target"(t2/update! :model/Transform transform-id {:source (update source :body str "\n")})- (assert-stale :transform transform-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :transform transform-id)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,@@ -453,13 +896,13 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty?(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))(deftest ^:synchronized query-transform-update-handles-downstream-dependencies-test(testing "query transform updates handle downstream dependencies"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders"):type :query}@@ -476,8 +919,8 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "keeping target"(t2/update! :model/Transform transform-id {:source (assoc source :query (lib/native-query mp "select * from orders limit 10"))})- (assert-stale :transform transform-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :transform transform-id)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,@@ -485,12 +928,12 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty?(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))(deftest segment-update-sets-correct-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)@@ -499,7 +942,7 @@:definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a segment creates dependency to its table"(assert-pending :segment segment-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :segment:from_entity_id segment-id:to_entity_type :table@@ -508,8 +951,8 @@(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))(testing "updating segment definition recalculates dependencies"(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})- (assert-stale :segment segment-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :segment segment-id)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :segment:from_entity_id segment-id:to_entity_type :table@@ -518,23 +961,23 @@(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))(testing "deleting segment removes all dependencies"(t2/delete! :model/Segment segment-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))(deftest card-with-segment-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)](mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id:definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a card using a segment creates dependencies to both segment and table"- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(mt/with-temp [:model/Card {card-id :id} {:dataset_query {:database (mt/id):type :query:query {:source-table products-id:filter [:segment segment-id]}}}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :card:from_entity_id card-id:to_entity_type :segment@@ -547,7 +990,7 @@(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))(deftest measure-update-sets-correct-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](mt/with-test-user :rasta(let [orders-id (mt/id :orders)@@ -559,7 +1002,7 @@(lib/aggregate (lib/sum quantity)))}](testing "creating a measure creates dependency to its table"(assert-pending :measure measure-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :measure:from_entity_id measure-id:to_entity_type :table@@ -569,8 +1012,8 @@(testing "updating measure definition recalculates dependencies"(t2/update! :model/Measure measure-id {:definition (-> (lib/query mp orders)(lib/aggregate (lib/sum subtotal)))})- (assert-stale :measure measure-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :measure measure-id)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :measure:from_entity_id measure-id:to_entity_type :table@@ -579,11 +1022,11 @@(t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))(testing "deleting measure removes all dependencies"(t2/delete! :model/Measure measure-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))(deftest action-update-sets-correct-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [action-deps (fn [action-id](t2/select-fn-set (juxt :to_entity_type :to_entity_id)@@ -594,19 +1037,19 @@:database_id (mt/id):dataset_query (sql-query "venues")}](testing "creating an action creates dependencies on the tables its query reads"- (assert-stale :action action-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :action action-id)+ (deps.test/synchronously-run-dependency-job!)(is (= #{[:table (mt/id :venues)]}(action-deps action-id))))(testing "updating an action's query recalculates its dependencies"(t2/update! :model/QueryAction action-id {:dataset_query (sql-query "checkins")})- (assert-stale :action action-id)- (deps.test/synchronously-run-backfill!)+ (assert-pending :action action-id)+ (deps.test/synchronously-run-dependency-job!)(is (= #{[:table (mt/id :checkins)]}(action-deps action-id))))(testing "deleting an action removes its dependencies and status"(t2/delete! :model/Action action-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (empty? (action-deps action-id)))(is (not (t2/exists? :model/DependencyStatus :entity_type :action :entity_id action-id)))))))))@@ -619,7 +1062,7 @@:entity_type :action :entity_id [:in action-ids]))})(deftest model-delete-removes-dependency-rows-of-its-actions-test- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](mt/with-temp [:model/Card {model-id :id} {:type :model:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}@@ -629,18 +1072,18 @@:model/QueryAction _ {:action_id query-id:database_id (mt/id):dataset_query (lib/native-query mp "DELETE FROM venues WHERE id = 1")}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "the model's actions have dependency rows"(is (= {:dependencies #{implicit-id query-id}, :statuses #{implicit-id query-id}}(action-dependency-rows [implicit-id query-id]))))(testing "deleting the model removes the dependency rows of its actions"(t2/delete! :model/Card model-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= {:dependencies #{}, :statuses #{}}(action-dependency-rows [implicit-id query-id]))))))))(deftest database-delete-removes-dependency-rows-of-its-models-actions-test- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [_mp](mt/with-temp [:model/Database {db-id :id} {:engine :h2, :details (:details (mt/db))}:model/Table {table-id :id} {:db_id db-id, :name "VENUES", :schema "PUBLIC"}]@@ -654,19 +1097,19 @@:model/QueryAction _ {:action_id query-id:database_id db-id:dataset_query (lib/native-query db-mp "DELETE FROM venues WHERE id = 1")}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "the actions of the database's model have status rows"(is (= #{implicit-id query-id}(:statuses (action-dependency-rows [implicit-id query-id])))))(testing "deleting the database removes the dependency rows of the actions of its models"(t2/delete! :model/Database db-id)- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= {:dependencies #{}, :statuses #{}}(action-dependency-rows [implicit-id query-id]))))(is (some? table-id))))))))(deftest measure-with-measure-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)@@ -681,7 +1124,7 @@(lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)(lib/sum subtotal))))}](testing "creating a measure that references another measure creates dependencies to both"- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :measure:from_entity_id measure2-id:to_entity_type :table@@ -694,7 +1137,7 @@(t2/select :model/Dependency :from_entity_id measure2-id :from_entity_type :measure)))))))))))))(deftest card-with-measure-dependencies- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)@@ -702,13 +1145,13 @@(mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id:definition (-> (lib/query mp orders)(lib/aggregate (lib/sum quantity)))}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "creating a card using a measure creates dependencies to both measure and table"(let [mp' (mt/metadata-provider)query (-> (lib/query mp' orders)(lib/aggregate (lib.metadata/measure mp' measure-id)))](mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{{:from_entity_type :card:from_entity_id card-id:to_entity_type :measure@@ -720,388 +1163,37 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card)))))))))))))-;; === Analysis finding tests (these are unchanged — they use AnalysisFinding, not dependency_analysis_version) ===--(defn- assert-has-analyses- "Asserts that a given set of entities have the appropriate analysis version.-- Format is `{entity-type {entity-id analysis-version}}`.-- Use -1 for analysis-version if you want to check that an analysis exists without asserting a specific version.-- Use nil for analysis-version if you want to check that no analysis exists."- [spec]- (doseq [[entity-type ids-and-versions] spec]- (let [analyses-map (->> (t2/select [:model/AnalysisFinding :analyzed_entity_id :analysis_version]- :analyzed_entity_type entity-type)- (into {} (map (juxt :analyzed_entity_id :analysis_version))))]- (doseq [[id expected-version] ids-and-versions]- (testing (str "Checking " entity-type " " id)- (if (= expected-version -1)- (is (contains? analyses-map id))- (is (= expected-version (analyses-map id)))))))))--(deftest ^:synchronized card-update-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Card updates should mark the card stale and trigger the background job"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}- :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-card-id- :to_entity_type :card- :to_entity_id parent-card-id}]- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})- ;; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities- (assert-has-analyses- {:card {parent-card-id nil}})- (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-card-id))- "entities without analysis findings should not be marked: no fake analysis record should be created")))))))--(deftest ^:synchronized card-update-updates-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Card updates should mark the card stale, then the job re-analyzes and propagates"- (let [products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-card-id- :to_entity_type :card- :to_entity_id parent-card-id}]- (deps.findings/upsert-analysis! parent-card)- (deps.findings/upsert-analysis! child-card)- (deps.findings/upsert-analysis! other-card)- (deps.dependency-status/upsert-status! :card child-card-id false)- (deps.dependency-status/upsert-status! :card other-card-id false)- (testing "Checking that the initial analyses exist"- (assert-has-analyses- {:card {parent-card-id old-version- child-card-id old-version- other-card-id old-version}}))- (testing "changing the card query marks its finding and its dependents' stale, not unrelated ones"- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})- (deps.test/synchronously-run-backfill!)- (assert-has-analyses- {:card {parent-card-id old-version- child-card-id -1- other-card-id old-version}})- (is (= {parent-card-id true child-card-id true other-card-id false}- (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding :analyzed_entity_type :card- :analyzed_entity_id [:in [parent-card-id child-card-id other-card-id]]))))- (testing "If the version changes, the job picks up only the stale cards"- (binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)]- (assert-has-analyses- {:card {parent-card-id old-version- child-card-id -1- other-card-id old-version}})))))))))--;;; ------------------------------------------------ Analysis propagation tests --------------------------------------------------(deftest ^:synchronized card-update-triggers-native-cards-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Card update marks the card and its transitive dependents stale, and the entity-check job re-analyzes them"- (mt/with-temp [:model/Card {parent-id :id :as parent} {:dataset_query (lib/native-query mp "select * from products")}- :model/Card {child-id :id :as child} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}- :model/Dependency _ {:from_entity_type :card :from_entity_id child-id- :to_entity_type :card :to_entity_id parent-id}]- (deps.findings/upsert-analysis! parent)- (deps.findings/upsert-analysis! child)- (deps.dependency-status/upsert-status! :card child-id false)- (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})- (deps.test/synchronously-run-backfill!)- (testing "Parent card should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id parent-id))))- ;; Run entity-check job to process stale entities- (#'task.entity-check/check-entities!)- (testing "Parent should be re-analyzed"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id parent-id))))- (testing "Child should have been re-analyzed (it was marked stale up front as a transitive dependent)"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-id)))))))))--(deftest ^:synchronized card-update-stops-on-transforms-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Card update marks the card and its transitive dependents (including transforms) stale up front"- (let [products (lib.metadata/table mp (mt/id :products))- orders (lib.metadata/table mp (mt/id :orders))- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :transform :from_entity_id transform-id- :to_entity_type :card :to_entity_id parent-card-id}- :model/Dependency _ {:from_entity_type :card :from_entity_id child-card-id- :to_entity_type :transform :to_entity_id transform-id}]- (deps.findings/upsert-analysis! parent-card)- (deps.findings/upsert-analysis! transform)- (deps.findings/upsert-analysis! child-card)- (assert-has-analyses- {:card {parent-card-id old-version child-card-id old-version}- :transform {transform-id old-version}})- (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})- ;; Run entity-check job to drain the stale entities- (#'task.entity-check/check-entities!)- (testing "All entities should be re-analyzed after the drain"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id parent-card-id)))- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform :analyzed_entity_id transform-id)))- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id child-card-id))))))))))--(deftest ^:synchronized transform-update-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Transform update does not error when no analyses exist yet"- (let [products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- (is (= 1 (t2/update! :model/Transform transform-id- {:source {:type :query- :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))--(deftest ^:synchronized transform-update-updates-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Transform update marks entity stale, entity-check job re-analyzes it"- (let [products (lib.metadata/table mp (mt/id :products))- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- (deps.findings/upsert-analysis! transform)- (assert-has-analyses {:transform {transform-id old-version}})- (t2/update! :model/Transform transform-id {:source {:type :query- :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})- (deps.test/synchronously-run-backfill!)- (testing "Transform should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))))- (#'task.entity-check/check-entities!)- (testing "Transform should be re-analyzed"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))))))))))--(deftest ^:synchronized transform-update-triggers-native-transforms-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Native transform updates do not error"- (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query- :query (lib/native-query mp "select * from products")}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- (is (= 1 (t2/update! :model/Transform transform-id- {:source {:type :query- :query (lib/native-query mp "select * from orders")}}))))))))--(deftest ^:synchronized transform-run-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Transform run completes without error when no analyses exist"- (let [products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public" :name "sample" :type :table}}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :transform :to_entity_id transform-id}]- (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))--(deftest ^:synchronized transform-run-marks-dependents-stale-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Transform run marks immediate dependents stale"- (let [products (lib.metadata/table mp (mt/id :products))- orders (lib.metadata/table mp (mt/id :orders))]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id :as transform} {:source {:type :query :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public" :name "sample" :type :table}}- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :transform :to_entity_id transform-id}]- (deps.findings/upsert-analysis! card)- (deps.findings/upsert-analysis! transform)- (deps.findings/upsert-analysis! other-card)- (deps.hooks/after-transform-run transform-id nil)- (testing "Dependent card should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))- (testing "Other card should be unchanged"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))- (testing "Transform should be unchanged"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :transform :analyzed_entity_id transform-id))))))))))--(deftest ^:synchronized segment-update-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [_]- (testing "Segment update does not error when no analyses exist yet"- (let [products-id (mt/id :products)- price-field-id (mt/id :products :price)]- (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}]- (is (= 1 (t2/update! :model/Segment segment-id- {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))--(deftest ^:synchronized segment-update-updates-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Segment update marks entity stale, entity-check job re-analyzes and propagates"- (let [products-id (mt/id :products)- price-field-id (mt/id :products :price)- products (lib.metadata/table mp products-id)- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}- :model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :segment :to_entity_id segment-id}]- (deps.findings/upsert-analysis! segment)- (deps.findings/upsert-analysis! card)- (deps.dependency-status/upsert-status! :card card-id false)- (assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})- (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})- (deps.test/synchronously-run-backfill!)- (testing "Segment should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :segment :analyzed_entity_id segment-id))))- (#'task.entity-check/check-entities!)- (testing "Both segment and dependent card should be re-analyzed"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :segment :analyzed_entity_id segment-id)))- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))))))))--(deftest ^:synchronized table-metadata-update-triggers-dependent-analysis-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Table metadata update marks immediate dependents stale"- (let [products-id (mt/id :products)- orders (lib.metadata/table mp (mt/id :orders))- products (lib.metadata/table mp products-id)- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id products-id}]- (deps.findings/upsert-analysis! card)- (deps.findings/upsert-analysis! other-card)- (assert-has-analyses {:card {card-id old-version other-card-id old-version}})- (deps.test/do-with-immediate-hook-writes- #(mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]- (testing "Card depending on updated table should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))- (testing "Card not depending on updated table should not be stale"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))--(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Table metadata update does not error when no findings exist yet"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id products-id}]- (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]- (assert-has-analyses {:card {card-id nil}}))))))))--(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Field metadata update marks dependents of the field's table stale"- (let [products-id (mt/id :products)- orders (lib.metadata/table mp (mt/id :orders))- products (lib.metadata/table mp products-id)- old-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id products-id}]- (deps.findings/upsert-analysis! card)- (deps.findings/upsert-analysis! other-card)- (assert-has-analyses {:card {card-id old-version other-card-id old-version}})- (deps.test/do-with-immediate-hook-writes- #(mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]- (testing "Card depending on the table whose field was updated should be stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))- (testing "Card not depending on that table should not be stale"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))--(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Field metadata update does not error when no findings exist yet"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card :from_entity_id card-id- :to_entity_type :table :to_entity_id products-id}]- (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]- (assert-has-analyses {:card {card-id nil}}))))))))-(deftest serdes-load-recalculates-changed-entities-test(testing "a serdes load recalculates the dependencies of the entities it changes, and only those"- (run-with-dependencies-setup!+ (deps.test/run-with-dependencies-setup!(fn [mp](let [card-deps #(t2/select-fn-set (juxt :to_entity_type :to_entity_id):model/Dependency :from_entity_type :card :from_entity_id %)query #(lib/query mp (lib.metadata/table mp (mt/id %)))- processed #(t2/exists? :model/DependencyStatus :entity_type :card :entity_id % :stale false)load! #(serdes/with-cache(serdes.load/load-metabase! (ts/ingestion-in-memory %) :reindex? false))](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (query :products)}:model/Card {other-id :id} {:dataset_query (query :products)}](let [exported (serdes/extract-one "Card" {} card)](t2/update! :model/Card card-id {:dataset_query (query :orders)})- (deps.dependency-status/mark-stale! {:card #{card-id other-id}})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (= #{[:table (mt/id :orders)]} (card-deps card-id)))(load! [exported])- (testing "only the changed card is marked stale"- (assert-stale :card card-id)- (is (processed other-id)))- (deps.test/synchronously-run-backfill!)- (is (= #{[:table (mt/id :products)]} (card-deps card-id)))- (testing "reloading it unchanged marks nothing"- (load! [exported])- (is (processed card-id))))))))))+ (testing "only the changed card is pending"+ (assert-pending :card card-id)+ (is (not (pending? :card other-id))))+ (deps.test/synchronously-run-dependency-job!)+ (is (= #{[:table (mt/id :products)]} (card-deps card-id))))))))))++(deftest ^:synchronized new-upstream-marks-analysis-test+ (testing "storing a new upstream of an entity marks its analysis stale, since one done before the edge missed the+ upstream's changes"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFindingStatus :model/DependencyStatus]+ (mt/with-temp [:model/Card {source-id :id} {:dataset_query (deps.test/table-query :orders)}+ :model/Card {card-id :id} {:dataset_query (deps.test/table-query :venues)}]+ (deps.test/synchronously-run-dependency-job!)+ (t2/update! :model/Card card-id {:dataset_query (deps.test/card-query source-id)})+ (deps.test/record-analysis-finding-processed! :card card-id)+ (deps.test/synchronously-run-dependency-job!)+ (is (deps.test/analysis-finding-pending? :card card-id)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.cljdeleted file mode 100644index aa770887e24..00000000000--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj+++ /dev/null@@ -1,75 +0,0 @@-(ns metabase-enterprise.dependencies.task.entity-check-test- (:require- [clojure.test :refer [deftest is testing]]- [environ.core :as env]- [metabase-enterprise.dependencies.task.entity-check :as dependencies.entity-check]- [metabase.dependencies.hooks :as dependencies.hooks]- [metabase.premium-features.core :as premium-features]- [metabase.task.core :as task]- [metabase.test :as mt])- (:import- (org.quartz JobDetail)))--(set! *warn-on-reflection* true)--(deftest ^:synchronized batch-size-zero-suppresses-event-triggers-but-stays-warm-test- (testing "A non-positive batch size stops the job doing work, but leaves it on its slow periodic schedule so it- resumes if the setting ever becomes positive. What it must not do is keep firing the 1-second event-driven- trigger: entity changes fire it, and it never consulted the batch size, so a disabled job woke roughly once- a second on a busy instance."- (let [scheduled (atom [])- scheduled-by (fn [thunk]- (reset! scheduled [])- (thunk)- (count @scheduled))]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]- (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "0")]- (testing "the 1-second event-driven trigger is suppressed"- (is (zero? (scheduled-by dependencies.entity-check/trigger-entity-check-job!))))- (testing "but task/init! still puts the job on its periodic schedule"- (is (= 1 (scheduled-by #(task/init! ::dependencies.entity-check/DependencyEntityCheck)))))- (testing "and a run keeps the periodic chain going"- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly true)]- (is (= 1 (scheduled-by #(#'dependencies.entity-check/reschedule-after-run! nil)))))))- (testing "a positive batch size schedules from the event trigger too"- (with-redefs [env/env (assoc env/env :mb-dependency-entity-check-batch-size "5")]- (is (= 1 (scheduled-by dependencies.entity-check/trigger-entity-check-job!)))))))))--(deftest ^:synchronized post-run-reschedule-licence-states-test- (testing "The entity check job is a periodic checker with no terminal state, so it always queues another run. That- is right while licensed, but check-entities! is gated on :dependencies, so without the feature the job- rescheduled itself every 30-90 minutes forever to do nothing at all.-- Scheduling resolves the licence with canonically-has-feature?, not has-feature?, so it can distinguish- 'definitively unlicensed' from 'could not tell'. has-feature? collapses both to false, and since nothing- re-fires when a failed token check later succeeds, reading a blip as unlicensed would end the chain for- good. Only a definitive false stops it."- (let [scheduled (atom [])- scheduled-by (fn [licence]- (reset! scheduled [])- (mt/with-dynamic-fn-redefs [premium-features/canonically-has-feature? (constantly licence)]- (#'dependencies.entity-check/reschedule-after-run! nil))- (count @scheduled))]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]- (testing "licensed: keeps itself scheduled"- (is (= 1 (scheduled-by true))))- (testing "indeterminate: keeps the chain alive rather than ending it on a transient failure"- (is (= 1 (scheduled-by nil))))- (testing "definitively unlicensed: stops"- (is (zero? (scheduled-by false))))))))--(def ^:private entity-check-job-name "metabase.dependencies.task.entity-check.job")--(defn- scheduled-job-names- "Job names from captured [[task/schedule-task!]] calls, since a token update schedules both dependency jobs."- [calls]- (into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))--(deftest ^:synchronized token-update-restarts-the-job-test- (testing "Setting a premium token must restart the entity check job, or enabling the feature on an idle instance- leaves the periodic checker dead until the next process restart."- (let [scheduled (atom [])]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]- (mt/with-premium-features #{:dependencies}- (dependencies.hooks/after-premium-token-update)- (is (contains? (scheduled-job-names @scheduled) entity-check-job-name)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.cljnew file mode 100644index 00000000000..d1325645624--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/util_test.clj@@ -0,0 +1,44 @@+(ns metabase-enterprise.dependencies.task.util-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.dependencies.task.result-metadata :as task.result-metadata]+ [metabase-enterprise.dependencies.task.util :as task.util]+ [metabase.task.core :as task]+ [metabase.test :as mt])+ (:import+ (java.util Date)+ (org.quartz Scheduler TriggerKey)))++(set! *warn-on-reflection* true)++(defn- seconds-until-next-run ^long []+ (let [fire-time (.getNextFireTime (.getTrigger ^Scheduler (task/scheduler)+ (TriggerKey. "metabase.dependencies.task.result-metadata.trigger")))]+ (quot (- (.getTime ^Date fire-time) (System/currentTimeMillis)) 1000)))++(deftest schedule-run-keeps-earlier-run-test+ (testing "scheduling a later run keeps a run already pending earlier, and an earlier run replaces a later one"+ (mt/with-temp-scheduler!+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600)+ (is (< 3000 (seconds-until-next-run)))+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)+ (is (< (seconds-until-next-run) 61))+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600)+ (is (< (seconds-until-next-run) 61)))))++(deftest schedule-run-leaves-earlier-run-alone-test+ (testing "scheduling a later run while an earlier one is pending does not replace the pending trigger"+ (mt/with-temp-scheduler!+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)+ (let [scheduled (atom 0)]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& _] (swap! scheduled inc))]+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 60)+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600))+ (is (zero? @scheduled))))))++(deftest schedule-run-moves-past-due-run-test+ (testing "scheduling a run moves a run pending in the past, which Quartz may not fire for minutes, to a second from now"+ (mt/with-temp-scheduler!+ (mt/with-dynamic-fn-redefs [task.util/next-fire-time (constantly (Date. (- (.getTime (Date.)) (* 20 60 1000))))]+ (task.util/schedule-run! (task/scheduler) @#'task.result-metadata/job 3600))+ (is (<= 0 (seconds-until-next-run) 1)))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.cljindex a61b623fa59..03119603223 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj@@ -1,21 +1,114 @@(ns metabase-enterprise.dependencies.test-util(:require[medley.core :as m]- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.result-metadata-status :as models.result-metadata-status]+ [metabase-enterprise.dependencies.task.analysis-finding :as task.analysis-finding]+ [metabase-enterprise.dependencies.task.dependency :as task.dependency][metabase.app-db.core :as mdb]+ [metabase.lib-be.core :as lib-be][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-util.metadata-providers.mock :as providers.mock]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [toucan2.core :as t2]))-(defn synchronously-run-backfill!- "Run the dependency backfill job synchronously, processing all stale/outdated entities- until there is nothing left to process. Use this in tests after operations that mark- entities stale (create/update events) and before asserting on dependency data."+(defn run-with-dependencies-setup!+ "Run `thunk` with the metadata provider of the test database, as Rasta and with the dependencies feature, each write+ committing on its own as it does between real requests."+ [thunk]+ (mt/test-helpers-set-global-values!+ (mt/with-test-user :rasta+ (mt/with-premium-features #{:dependencies}+ (lib-be/with-metadata-provider-cache+ (let [mp (mt/metadata-provider)]+ (thunk mp)))))))++(defn synchronously-run-dependency-job!+ "Run the dependency job synchronously, processing every changed or outdated entity until there is nothing left to+ process. Use this in tests after writes and before asserting on dependency data."[](mt/with-premium-features #{:dependencies}- (while (#'dependencies.backfill/backfill-dependencies!))))+ (while (#'task.dependency/update-dependencies!))))++(defn run-analysis-finding-job!+ "Run batches of the analysis finding job until one takes nothing, or up to 20 of them, returning whether the last one+ took anything."+ []+ (loop [runs 1]+ (let [took? (some? (#'task.analysis-finding/run-batch!))]+ (if (and took? (< runs 20))+ (recur (inc runs))+ took?))))++(defn table-query+ "A query of the test data Table named `table`."+ [table]+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id table)))))++(defn card-query+ "A query of the Card with `card-id`."+ [card-id]+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/card mp card-id))))++(defn- status-pending?+ "Whether the entity `entity-type` `entity-id` has no status in the status `model`, or one that counts a write and+ that the job has not given up on."+ [model entity-type entity-id]+ (not (t2/exists? model {:where [:and+ [:= :entity_type (name entity-type)]+ [:= :entity_id entity-id]+ [:or [:= :stale 0] [:= :terminal true]]]})))++(defn record-status-processed!+ "Record, as a job does when it clears a status, that the entity `entity-type` `entity-id` was processed by the logic+ of `version`, inserting its status in the status `model` when there is none."+ [model entity-type entity-id version]+ (when (zero? (t2/update! model :entity_type entity-type :entity_id entity-id {:stale 0 :analysis_version version}))+ (t2/insert! model {:entity_type entity-type :entity_id entity-id :stale 0 :analysis_version version})))++(defn dependency-pending?+ "Whether the dependency job is to compute the dependencies of the entity `entity-type` `entity-id`: it has no+ status, or one that counts a write."+ [entity-type entity-id]+ (status-pending? :model/DependencyStatus entity-type entity-id))++(defn record-dependencies-computed!+ "Record, as the dependency job does when it clears its status, that the dependencies of the entity `entity-type`+ `entity-id` are current."+ [entity-type entity-id]+ (record-status-processed! :model/DependencyStatus entity-type entity-id+ models.dependency/current-dependency-analysis-version))++(defn result-metadata-pending?+ "Whether the result metadata job is to re-derive the Card with `card-id`: it has no status, or one that counts a+ write."+ [card-id]+ (status-pending? :model/ResultMetadataStatus :card card-id))++(defn record-result-metadata-processed!+ "Record, as the result metadata job does when it clears its status, that the result metadata of the Card with+ `card-id` is current."+ [card-id]+ (record-status-processed! :model/ResultMetadataStatus :card card-id+ models.result-metadata-status/current-result-metadata-version))++(defn analysis-finding-pending?+ "Whether the analysis job is to analyze the entity `entity-type` `entity-id`: it has no status, or one that counts a+ write."+ [entity-type entity-id]+ (status-pending? :model/AnalysisFindingStatus entity-type entity-id))++(defn record-analysis-finding-processed!+ "Record, as the analysis job does when it clears its status, that the finding of the entity `entity-type`+ `entity-id` is current."+ [entity-type entity-id]+ (record-status-processed! :model/AnalysisFindingStatus entity-type entity-id+ models.analysis-finding/*current-analysis-finding-version*))(defn do-with-immediate-hook-writes"Run `thunk` with the work the dependency hooks defer until commit done at once, since a test's transaction neverdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.cljindex 6e4c065e2cf..4b7c76473a3 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj@@ -359,7 +359,7 @@(let [app (t2/select-one :model/DataApp :name "shop")](perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)(group-access/add-groups! app [(:id group)])- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}](group-access/permission-warnings app))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.cljindex f9fe5207c5a..0d3c259eb59 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj@@ -143,7 +143,7 @@;; Populate dependencies via events(doseq [card [old-model mbql-child-1 mbql-child-2 native-child grandchild grandchild-native]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id:source_entity_type :card@@ -173,7 +173,7 @@(let [src (t2/select-one-fn :source :model/Transform :id transform-id)](is (= new-id (lib/primary-source-card-id (:query src))))))(testing "Dependencies point to new model"- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [edges-to (fn [card-id](t2/select-fn-set (juxt :from_entity_type :from_entity_id):model/Dependency@@ -207,7 +207,7 @@:name "Child card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!);; Insert a fake active run to simulate one already running(let [run (replacement-run/create-run! :card old-id :card new-id (mt/user->id :crowberto))](replacement-run/start-run! (:id run)))@@ -239,7 +239,7 @@:name "Child Card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id:source_entity_type :card@@ -272,7 +272,7 @@:name "Child card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(snowplow-test/with-fake-snowplow-collector(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id@@ -388,7 +388,7 @@;; Populate dependencies via events(doseq [card [model-card question-card]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(snowplow-test/with-fake-snowplow-collector(let [response (mt/user-http-request :crowberto :post 202"ee/replacement/replace-model-with-transform"@@ -509,7 +509,7 @@(mt/with-model-cleanup [:model/ReplacementRun](doseq [card [model-card direct-card nested-card metric-card join-card]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [response (mt/user-http-request :crowberto :post 202"ee/replacement/replace-model-with-transform"{:card_id model-iddiff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.cljindex 61704ca4d83..960cdc4ef0a 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj@@ -189,7 +189,7 @@;; populate dependencies(events/publish-event! :event/card-create {:object old-card :user-id (mt/user->id :rasta)})(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :rasta)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(testing "child card initially points to old model"(is (= old-id (get-in (t2/select-one-fn :dataset_query :model/Card :id child-id)[:stages 0 :source-card]))))@@ -243,7 +243,7 @@(mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](doseq [card [old-card child-1 child-2]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :rasta)}))- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(let [original-swap! (mt/original-fn #'replacement.source-swap/swap-source!)](mt/with-dynamic-fn-redefs [replacement.source-swap/swap-source!(fn [entity object old-source new-source]diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.cljindex bdb98152c40..ad757017ed0 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj@@ -419,7 +419,7 @@;; Create a dependency: card -> orders table(events/publish-event! :event/card-create {:object (t2/select-one :model/Card card-id):user-id (mt/user->id :rasta)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders))@@ -449,7 +449,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/segment-create {:object (t2/select-one :model/Segment segment-id):user-id (mt/user->id :rasta)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency:from_entity_type :segment :from_entity_id segment-id:to_entity_type :table :to_entity_id (mt/id :orders)))@@ -479,7 +479,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/measure-create {:object (t2/select-one :model/Measure measure-id):user-id (mt/user->id :rasta)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency:from_entity_type :measure :from_entity_id measure-id:to_entity_type :table :to_entity_id (mt/id :orders)))@@ -508,7 +508,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/transform-create {:object (t2/select-one :model/Transform transform-id):user-id (mt/user->id :rasta)})- (deps.test/synchronously-run-backfill!)+ (deps.test/synchronously-run-dependency-job!)(is (t2/exists? :model/Dependency:from_entity_type :transform :from_entity_id transform-id:to_entity_type :table :to_entity_id (mt/id :orders)))diff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.cljindex 487366aa058..b488d7ec8a9 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj@@ -76,7 +76,7 @@(mt/with-model-cleanup [:model/Card :model/Dependency](let [source-card (card/create-card! (card-with-query "Source card" :products) user)child-card (card/create-card! (card-sourced-from "Child card" source-card) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)found-usages (replacement.usages/transitive-usages [:card (:id source-card)])](is (some #(= [:card (:id child-card)] %) found-usages)"Child card should appear in usages"))))))))@@ -90,7 +90,7 @@(let [source-card (card/create-card! (card-with-query "Source card" :products) user)child-card (card/create-card! (card-sourced-from "Child card" source-card) user)grandchild-card (card/create-card! (card-sourced-from "Grandchild card" child-card) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)found-usages (replacement.usages/transitive-usages [:card (:id source-card)])](is (some #(= [:card (:id child-card)] %) found-usages)"Child card should appear in usages")@@ -104,7 +104,7 @@(let [user (mt/fetch-user :rasta)](mt/with-model-cleanup [:model/Card :model/Dependency](let [lonely-card (card/create-card! (card-with-query "Lonely card" :products) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)found-usages (replacement.usages/transitive-usages [:card (:id lonely-card)])](is (empty? found-usages)"Card with no dependents should have empty usages"))))))))@@ -116,7 +116,7 @@(let [user (mt/fetch-user :rasta)](mt/with-model-cleanup [:model/Card :model/Dependency](let [card-on-table (card/create-card! (card-with-query "Card on products" :products) user)- _ (deps.test/synchronously-run-backfill!)+ _ (deps.test/synchronously-run-dependency-job!)found-usages (replacement.usages/transitive-usages [:table (mt/id :products)])](is (some #(= [:card (:id card-on-table)] %) found-usages)"Card using table should appear in usages"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 120e6e3f686..6be8f4f449d 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -37,6 +37,7 @@:model/AiUsageLog:model/AnalysisFinding:model/AnalysisFindingError+ :model/AnalysisFindingStatus:model/ApiKey:model/ApiKeyUsageLog:model/AuthIdentity@@ -122,6 +123,7 @@:model/ReplacementRun:model/RemoteSyncObject:model/RemoteSyncTask+ :model/ResultMetadataStatus:model/Revision:model/SemanticSearchTokenTracking:model/SearchIndexMetadatadiff --git a/resources/migrations/065/20261018_dependency_change_detection.yaml b/resources/migrations/065/20261018_dependency_change_detection.yamlnew file mode 100644index 00000000000..b7cbbd6f7cb--- /dev/null+++ b/resources/migrations/065/20261018_dependency_change_detection.yaml@@ -0,0 +1,503 @@+## The dependency jobs process the entities whose status rows a write marks stale in its own transaction: an+## entity's own row for the columns each job reads, and its direct dependents' rows when what they read changes. The+## status rows lose the retry delays and keep a failure count and a terminal flag; their stale flag becomes a count+## of the writes the job has not read. result_metadata_status and analysis_finding_status hold one row per entity,+## as dependency_status does, and analysis_finding keeps only the findings.++databaseChangeLog:+ - changeSet:+ id: v65.drkztr+ author: ranquild+ comment: Drop dependency_status.next_retry_at, which the jobs no longer use since they don't retry+ changes:+ - dropColumn:+ tableName: dependency_status+ columnName: next_retry_at+ rollback:+ - addColumn:+ tableName: dependency_status+ columns:+ - column:+ name: next_retry_at+ type: ${timestamp_type}+ remarks: Earliest time at which a failed entity can be retried (null means immediately processable)++ - changeSet:+ id: v65.u3r8zm+ author: ranquild+ comment: Drop dependency_status.unresolved_tables, since a table a native query names is recorded once the query changes again+ changes:+ - dropColumn:+ tableName: dependency_status+ columnName: unresolved_tables+ rollback:+ - addColumn:+ tableName: dependency_status+ columns:+ - column:+ name: unresolved_tables+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity names tables that match no synced table, recomputed after each sync+ constraints:+ nullable: false++ - changeSet:+ id: v65.lywci7+ author: ranquild+ comment: Rename dependency_status.dependency_analysis_version to analysis_version, as the other status tables name it+ changes:+ - renameColumn:+ tableName: dependency_status+ oldColumnName: dependency_analysis_version+ newColumnName: analysis_version+ columnDataType: smallint+ rollback:+ - renameColumn:+ tableName: dependency_status+ oldColumnName: analysis_version+ newColumnName: dependency_analysis_version+ columnDataType: smallint+ - addNotNullConstraint:+ tableName: dependency_status+ columnName: dependency_analysis_version+ columnDataType: smallint+ - addDefaultValue:+ tableName: dependency_status+ columnName: dependency_analysis_version+ columnDataType: smallint+ defaultValueNumeric: 0++ - changeSet:+ id: v65.0mshmu+ author: ranquild+ comment: Keep dependency_status.analysis_version not null, which a rename on MySQL drops+ changes:+ - addNotNullConstraint:+ tableName: dependency_status+ columnName: analysis_version+ columnDataType: smallint+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.wlb1z1+ author: ranquild+ comment: Keep the default of dependency_status.analysis_version, which a rename on MySQL drops+ changes:+ - addDefaultValue:+ tableName: dependency_status+ columnName: analysis_version+ columnDataType: smallint+ defaultValueNumeric: 0+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.oacw7z+ author: ranquild+ comment: Create result_metadata_status+ changes:+ - createTable:+ tableName: result_metadata_status+ remarks: Whether the result metadata of each MBQL card is to be re-derived+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: entity_type+ type: varchar(254)+ remarks: The type of the entity+ constraints:+ nullable: false+ - column:+ name: entity_id+ type: int+ remarks: The ID of the entity+ constraints:+ nullable: false+ - column:+ name: stale+ type: int+ defaultValueNumeric: 0+ remarks: How many writes changed what the job reads for the entity since the job last read it; the job processes it while above 0+ constraints:+ nullable: false+ - column:+ name: fail_count+ type: int+ defaultValueNumeric: 0+ remarks: How many times in a row the job failed to process the entity since a write changed it+ constraints:+ nullable: false+ - column:+ name: terminal+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the job gave up on the entity after its failures, until a write changes it+ constraints:+ nullable: false+ - column:+ name: analysis_version+ type: int+ remarks: The version of the result metadata logic that last processed the card+ constraints:+ nullable: false+ rollback:+ - dropTable:+ tableName: result_metadata_status++ - changeSet:+ id: v65.ckqf9u+ author: ranquild+ comment: Unique constraint to prevent duplicate result_metadata_status entries+ changes:+ - addUniqueConstraint:+ tableName: result_metadata_status+ columnNames: entity_type, entity_id+ constraintName: idx_unique_result_metadata_status+ rollback:+ - dropUniqueConstraint:+ tableName: result_metadata_status+ constraintName: idx_unique_result_metadata_status++ - changeSet:+ id: v65.u44qob+ author: ranquild+ comment: Create analysis_finding_status+ changes:+ - createTable:+ tableName: analysis_finding_status+ remarks: Whether each analyzable entity is to be analyzed again+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: entity_type+ type: varchar(254)+ remarks: The type of the entity+ constraints:+ nullable: false+ - column:+ name: entity_id+ type: int+ remarks: The ID of the entity+ constraints:+ nullable: false+ - column:+ name: stale+ type: int+ defaultValueNumeric: 0+ remarks: How many writes changed what the job reads for the entity since the job last read it; the job processes it while above 0+ constraints:+ nullable: false+ - column:+ name: fail_count+ type: int+ defaultValueNumeric: 0+ remarks: How many times in a row the job failed to process the entity since a write changed it+ constraints:+ nullable: false+ - column:+ name: terminal+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the job gave up on the entity after its failures, until a write changes it+ constraints:+ nullable: false+ - column:+ name: analysis_version+ type: int+ remarks: The version of the analysis that last processed the entity+ constraints:+ nullable: false+ rollback:+ - dropTable:+ tableName: analysis_finding_status++ - changeSet:+ id: v65.0ypnl2+ author: ranquild+ comment: Unique constraint to prevent duplicate analysis_finding_status entries+ changes:+ - addUniqueConstraint:+ tableName: analysis_finding_status+ columnNames: entity_type, entity_id+ constraintName: idx_unique_analysis_finding_status+ rollback:+ - dropUniqueConstraint:+ tableName: analysis_finding_status+ constraintName: idx_unique_analysis_finding_status++ - changeSet:+ id: v65.eoug8w+ author: ranquild+ comment: Copy the stale flag and analysis version of each analysis_finding row to analysis_finding_status+ changes:+ - sql:+ sql: >-+ INSERT INTO analysis_finding_status (entity_type, entity_id, stale, analysis_version)+ SELECT analyzed_entity_type, analyzed_entity_id, CASE WHEN stale THEN 1 ELSE 0 END, COALESCE(analysis_version, 0)+ FROM analysis_finding+ rollback: # the table is dropped by the rollback of the changeset creating it++ - changeSet:+ id: v65.1suf0c+ author: ranquild+ comment: Drop the index on analysis_finding.stale, which moves to analysis_finding_status+ changes:+ - dropIndex:+ tableName: analysis_finding+ indexName: idx_analysis_finding_stale+ rollback:+ - sql:+ dbms: postgresql+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale) WHERE stale = true+ - sql:+ dbms: mysql,mariadb,h2+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale)++ - changeSet:+ id: v65.d291qd+ author: ranquild+ comment: Drop analysis_finding.stale, which analysis_finding_status now holds+ changes:+ - dropColumn:+ tableName: analysis_finding+ columnName: stale+ rollback:+ - addColumn:+ tableName: analysis_finding+ columns:+ - column:+ name: stale+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity needs re-analysis due to upstream changes+ constraints:+ nullable: false+ - sql:+ sql: >-+ UPDATE analysis_finding SET stale = COALESCE((SELECT s.stale > 0+ FROM analysis_finding_status s WHERE s.entity_type = analysis_finding.analyzed_entity_type+ AND s.entity_id = analysis_finding.analyzed_entity_id), FALSE)++ - changeSet:+ id: v65.em1iri+ author: ranquild+ comment: Drop analysis_finding.analysis_version, which analysis_finding_status records+ changes:+ - dropColumn:+ tableName: analysis_finding+ columnName: analysis_version+ rollback:+ - addColumn:+ tableName: analysis_finding+ columns:+ - column:+ name: analysis_version+ type: int+ defaultValueNumeric: 0+ remarks: The version of the analysis that was performed+ constraints:+ nullable: false+ - sql:+ sql: >-+ UPDATE analysis_finding SET analysis_version = COALESCE((SELECT s.analysis_version+ FROM analysis_finding_status s WHERE s.entity_type = analysis_finding.analyzed_entity_type+ AND s.entity_id = analysis_finding.analyzed_entity_id), 0)++ - changeSet:+ id: v65.pxgfgg+ author: ranquild+ comment: Rename analysis_finding.analyzed_entity_type to entity_type, as the status tables name it+ changes:+ - renameColumn:+ tableName: analysis_finding+ oldColumnName: analyzed_entity_type+ newColumnName: entity_type+ columnDataType: varchar(254)+ rollback:+ - renameColumn:+ tableName: analysis_finding+ oldColumnName: entity_type+ newColumnName: analyzed_entity_type+ columnDataType: varchar(254)+ - addNotNullConstraint:+ tableName: analysis_finding+ columnName: analyzed_entity_type+ columnDataType: varchar(254)++ - changeSet:+ id: v65.2rxyyj+ author: ranquild+ comment: Rename analysis_finding.analyzed_entity_id to entity_id, as the status tables name it+ changes:+ - renameColumn:+ tableName: analysis_finding+ oldColumnName: analyzed_entity_id+ newColumnName: entity_id+ columnDataType: int+ rollback:+ - renameColumn:+ tableName: analysis_finding+ oldColumnName: entity_id+ newColumnName: analyzed_entity_id+ columnDataType: int+ - addNotNullConstraint:+ tableName: analysis_finding+ columnName: analyzed_entity_id+ columnDataType: int++ - changeSet:+ id: v65.p5wigm+ author: ranquild+ comment: Keep analysis_finding.entity_type not null, which a rename on MySQL drops+ changes:+ - addNotNullConstraint:+ tableName: analysis_finding+ columnName: entity_type+ columnDataType: varchar(254)+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.j3y4t5+ author: ranquild+ comment: Keep analysis_finding.entity_id not null, which a rename on MySQL drops+ changes:+ - addNotNullConstraint:+ tableName: analysis_finding+ columnName: entity_id+ columnDataType: int+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.7d5dj3+ author: ranquild+ comment: Drop the index on dependency_status.stale, which becomes a count of the writes the job has not read+ changes:+ - dropIndex:+ tableName: dependency_status+ indexName: idx_dependency_status_stale+ rollback:+ - sql:+ dbms: postgresql+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale = true+ - sql:+ dbms: mysql,mariadb,h2+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)++ - changeSet:+ id: v65.47bljg+ author: ranquild+ comment: Add dependency_status.stale_count, which replaces the stale flag+ changes:+ - addColumn:+ tableName: dependency_status+ columns:+ - column:+ name: stale_count+ type: int+ defaultValueNumeric: 0+ remarks: How many writes changed what the job reads for the entity since the job last read it+ constraints:+ nullable: false+ rollback:+ - dropColumn:+ tableName: dependency_status+ columnName: stale_count++ - changeSet:+ id: v65.i33b3h+ author: ranquild+ comment: Copy the dependency_status stale flags into stale_count+ changes:+ - sql:+ sql: UPDATE dependency_status SET stale_count = CASE WHEN stale THEN 1 ELSE 0 END+ rollback:+ - sql:+ sql: UPDATE dependency_status SET stale = (stale_count > 0)++ - changeSet:+ id: v65.vcn86w+ author: ranquild+ comment: Drop the dependency_status stale flag, which stale_count replaces+ changes:+ - dropColumn:+ tableName: dependency_status+ columnName: stale+ rollback:+ - addColumn:+ tableName: dependency_status+ columns:+ - column:+ name: stale+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity needs (re)processing+ constraints:+ nullable: false++ - changeSet:+ id: v65.accafc+ author: ranquild+ comment: Rename dependency_status.stale_count to stale, as the other status tables name it+ changes:+ - renameColumn:+ tableName: dependency_status+ oldColumnName: stale_count+ newColumnName: stale+ columnDataType: int+ rollback:+ - renameColumn:+ tableName: dependency_status+ oldColumnName: stale+ newColumnName: stale_count+ columnDataType: int++ - changeSet:+ id: v65.cw6b2z+ author: ranquild+ comment: Keep dependency_status.stale not null, which a rename on MySQL drops+ changes:+ - addNotNullConstraint:+ tableName: dependency_status+ columnName: stale+ columnDataType: int+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.hn74zg+ author: ranquild+ comment: Keep the default of dependency_status.stale, which a rename on MySQL drops+ changes:+ - addDefaultValue:+ tableName: dependency_status+ columnName: stale+ columnDataType: int+ defaultValueNumeric: 0+ rollback: # the rename's rollback restores the column++ - changeSet:+ id: v65.f2ttgl+ author: ranquild+ comment: Index dependency_status.stale, the count of writes the job has not read+ changes:+ - sql:+ dbms: postgresql+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale > 0+ - sql:+ dbms: mysql,mariadb,h2+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)+ rollback:+ - dropIndex:+ tableName: dependency_status+ indexName: idx_dependency_status_stalediff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.cljindex 17a48fed6e4..89cfa4a0d34 100644--- a/src/metabase/batch_processing/impl.clj+++ b/src/metabase/batch_processing/impl.clj@@ -110,7 +110,7 @@sdk-tracked-rowid+timestamp[:map {:closed true}- [:table-id ms/PositiveInt]]+ [:dependency-jobs [:= true]]][:map {:closed true}[:host :string][:timestamp (ms/InstanceOfClass java.time.temporal.Temporal)]]diff --git a/src/metabase/dependencies/hooks.clj b/src/metabase/dependencies/hooks.cljindex c7504438bd7..1d863c42911 100644--- a/src/metabase/dependencies/hooks.clj+++ b/src/metabase/dependencies/hooks.clj@@ -1,50 +1,48 @@(ns metabase.dependencies.hooks- "Hooks that keep the dependency graph current as content is written, implemented by the enterprise module."+ "Hooks that have the dependency jobs triggered as content is written, implemented by the enterprise module."(:require[metabase.premium-features.defenterprise :refer [defenterprise]]))(defenterprise before-insert- "Hook for the `model` row `instance` before it is inserted. The OSS implementation is a no-op."+ "Hook for `row` of `model` before it is inserted, which does nothing without the dependencies feature."metabase-enterprise.dependencies.hooks- [_model _instance]+ [_model _row]nil)(defenterprise before-update- "Hook for the `model` row `instance` before it is updated, whose Toucan changes say what changes. The OSS- implementation is a no-op."+ "Hook for `row` of `model` before it is updated, which does nothing without the dependencies feature."metabase-enterprise.dependencies.hooks- [_model _instance]+ [_model _row]nil)(defenterprise before-delete- "Hook for the `model` row `instance` before it is deleted. The OSS implementation is a no-op."+ "Hook for `row` of `model` before it is deleted, which does nothing without the dependencies feature."metabase-enterprise.dependencies.hooks- [_model _instance]- nil)--(defenterprise after-database-sync- "Hook for the Database with `database-id` after a sync. The OSS implementation is a no-op."- metabase-enterprise.dependencies.hooks- [_database-id]- nil)--(defenterprise after-serdes-load- "Hook for after a serdes load, during which the hooks mark what the load changes without triggering the jobs. The- OSS implementation is a no-op."- metabase-enterprise.dependencies.hooks- []+ [_model _row]nil)(defenterprise after-premium-token-update- "Hook for after the premium token changed, which may have enabled the dependencies feature. The OSS implementation is- a no-op."+ "Hook for after the premium token changed, which may have enabled the dependencies feature, and does nothing without+ it."metabase-enterprise.dependencies.hooks[]nil)(defenterprise after-transform-run"Hook for the transform with `transform-id` after a run wrote its output Table with `table-id`, or nil when the- table was not synced. The OSS implementation is a no-op."+ table was not synced, which does nothing without the dependencies feature."metabase-enterprise.dependencies.hooks[_transform-id _table-id]nil)++(defenterprise do-with-batched-marks+ "Run `thunk`, with the dependents of what its writes change marked once each when it ends."+ metabase-enterprise.dependencies.hooks+ [thunk]+ (thunk))++(defmacro with-batched-marks+ "Run `body`, with the dependents of what its writes change marked once each when it ends, for a batch of writes+ such as a sync's."+ [& body]+ `(do-with-batched-marks (fn [] ~@body)))diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 0e6edf8dd3d..2cbc81bfff1 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -1612,7 +1612,7 @@to get the raw native query as needed.A table the query names that matches no synced table or transform target can be returned as- `{:unresolved-table {:schema ... :table ...}}`, so dependency tracking can recompute the query once sync finds it.+ `{:unresolved-table {:schema ... :table ...}}`, which dependency tracking does not record as an upstream.The return value should match the `:metabase.driver/native-query-deps` schema."{:added "0.57.0" :arglists '([driver query])}diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex dde7789d5ed..e3a5ed1a078 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -1582,6 +1582,7 @@metadata-provider?metadata-providerable?][metabase.lib.metadata.result-metadata+ match-returned-columnsnormalize-result-metadata-column][lib.nativeadd-parameters-for-template-tagsdiff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljcindex d782800ec7e..c01dbba6a88 100644--- a/src/metabase/lib/metadata/result_metadata.cljc+++ b/src/metabase/lib/metadata/result_metadata.cljc@@ -557,3 +557,16 @@(->> col(lib.normalize/normalize :metabase.query-processor.schema/result-metadata.column))):cljs (lib.normalize/normalize ::lib.schema.metadata/column col)))++(defn match-returned-columns+ "For each of the newly returned columns `cols`, the returned column of `stored-cols` it stands for, or nil: the one+ with the same desired column alias, or else the one with the same name when either has no alias."+ [stored-cols cols]+ (let [by-alias (m/index-by :lib/desired-column-alias (filter :lib/desired-column-alias stored-cols))+ by-name (m/index-by :name stored-cols)]+ (mapv (fn [col]+ (or (get by-alias (:lib/desired-column-alias col))+ (let [stored-col (get by-name (:name col))]+ (when (not-every? :lib/desired-column-alias [col stored-col])+ stored-col))))+ cols)))diff --git a/src/metabase/lib/schema/validate.cljc b/src/metabase/lib/schema/validate.cljcindex f6569de8ad1..d9060478b7a 100644--- a/src/metabase/lib/schema/validate.cljc+++ b/src/metabase/lib/schema/validate.cljc@@ -4,39 +4,39 @@[metabase.util.malli.registry :as mr]))(mr/def ::missing-column-error- [:map- [:type [:= :missing-column]]+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-column]][:name :string]])(mr/def ::missing-table-alias-error- [:map- [:type [:= :missing-table-alias]]+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-table-alias]][:name :string]])(mr/def ::missing-table-error"A stage's `:source-table` references a table that is missing or inactive (e.g. dropped during sync).The offending table is identified by the `:source-entity-*` keys, so no `:name` is carried."- [:map- [:type [:= :missing-table]]])+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-table]]])(mr/def ::missing-card-error"A stage's `:source-card` references a card that has been deleted.The offending card is identified by the `:source-entity-*` keys, so no `:name` is carried."- [:map- [:type [:= :missing-card]]])+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :missing-card]]])(mr/def ::duplicate-column-error- [:map- [:type [:= :duplicate-column]]+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :duplicate-column]][:name :string]])(mr/def ::syntax-error- [:map- [:type [:= :syntax-error]]])+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :syntax-error]]])(mr/def ::validation-exception-error- [:map- [:type [:= :validation-exception-error]]+ [:map {:closed true}+ [:type {:decode/normalize common/normalize-keyword} [:= :validation-exception-error]][:message :string]])(mr/def ::validate-error-type@@ -44,16 +44,14 @@:validation-exception-error])(mr/def ::error- [:and- [:map [:type {:decode/normalize common/normalize-keyword} ::validate-error-type]]- [:multi {:dispatch #(-> % :type keyword)}- [:missing-column [:ref ::missing-column-error]]- [:missing-table-alias [:ref ::missing-table-alias-error]]- [:missing-table [:ref ::missing-table-error]]- [:missing-card [:ref ::missing-card-error]]- [:duplicate-column [:ref ::duplicate-column-error]]- [:syntax-error [:ref ::syntax-error]]- [:validation-exception-error [:ref ::validation-exception-error]]]])+ [:multi {:dispatch #(some-> % :type keyword)}+ [:missing-column [:ref ::missing-column-error]]+ [:missing-table-alias [:ref ::missing-table-alias-error]]+ [:missing-table [:ref ::missing-table-error]]+ [:missing-card [:ref ::missing-card-error]]+ [:duplicate-column [:ref ::duplicate-column-error]]+ [:syntax-error [:ref ::syntax-error]]+ [:validation-exception-error [:ref ::validation-exception-error]]])(mr/def ::source-entity-type"The type of the source entity causing an error."@@ -61,23 +59,23 @@(mr/def ::source-entity"Optional source entity tracking fields for dependency analysis."- [:map+ [:map {:closed true}[:source-entity-type {:optional true} [:maybe ::source-entity-type]][:source-entity-id {:optional true} [:maybe pos-int?]]])+(mr/def ::soft+ "Whether the offending ref could be dropped, e.g. from a `:fields` list, without breaking the query."+ [:map {:closed true}+ [:soft? {:optional true} :boolean]])+(mr/def ::error-with-source"An error with optional source entity information.Extends the base error types with source tracking for dependency analysis."- [:and- [:map- [:type {:decode/normalize common/normalize-keyword} ::validate-error-type]- [:source-entity-type {:optional true} [:maybe ::source-entity-type]]- [:source-entity-id {:optional true} [:maybe pos-int?]]]- [:multi {:dispatch #(-> % :type keyword)}- [:missing-column [:merge [:ref ::missing-column-error] [:ref ::source-entity]]]- [:missing-table-alias [:merge [:ref ::missing-table-alias-error] [:ref ::source-entity]]]- [:missing-table [:merge [:ref ::missing-table-error] [:ref ::source-entity]]]- [:missing-card [:merge [:ref ::missing-card-error] [:ref ::source-entity]]]- [:duplicate-column [:merge [:ref ::duplicate-column-error] [:ref ::source-entity]]]- [:syntax-error [:merge [:ref ::syntax-error] [:ref ::source-entity]]]- [:validation-exception-error [:merge [:ref ::validation-exception-error] [:ref ::source-entity]]]]])+ [:multi {:dispatch #(some-> % :type keyword)}+ [:missing-column [:merge [:ref ::missing-column-error] [:ref ::source-entity] [:ref ::soft]]]+ [:missing-table-alias [:merge [:ref ::missing-table-alias-error] [:ref ::source-entity] [:ref ::soft]]]+ [:missing-table [:merge [:ref ::missing-table-error] [:ref ::source-entity] [:ref ::soft]]]+ [:missing-card [:merge [:ref ::missing-card-error] [:ref ::source-entity] [:ref ::soft]]]+ [:duplicate-column [:merge [:ref ::duplicate-column-error] [:ref ::source-entity] [:ref ::soft]]]+ [:syntax-error [:merge [:ref ::syntax-error] [:ref ::source-entity] [:ref ::soft]]]+ [:validation-exception-error [:merge [:ref ::validation-exception-error] [:ref ::source-entity] [:ref ::soft]]]])diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 8c56d7a6f94..9ec9ac31f52 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -694,7 +694,7 @@(methodical/defmethod t2.pipeline/transduce-query :around [:toucan.query-type/update.* :hook/dependencies :default]"Run the before-update hooks of a `:hook/dependencies` model in the transaction of its UPDATE, which Toucan otherwise- opens only after them, so dependency marks commit together with the UPDATE and after-commit work sees it."+ opens only after them, so the dependency marks commit or roll back with the UPDATE and after-commit work sees it."[rf query-type model parsed-args resolved-query](t2.conn/with-transaction [_conn nil {:nested-transaction-rule :ignore}](next-method rf query-type model parsed-args resolved-query)))diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 4bafbc3fc6b..13f231a1b50 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -16,6 +16,7 @@:model/AgentApiCallLog metabase.agent-api.models.agent-api-call-log:model/AnalysisFinding metabase-enterprise.dependencies.models.analysis-finding:model/AnalysisFindingError metabase-enterprise.dependencies.models.analysis-finding-error+ :model/AnalysisFindingStatus metabase-enterprise.dependencies.models.analysis-finding-status:model/ApiKey metabase.api-keys.models.api-key:model/ApiKeyUsageLog metabase.api-keys.models.api-key-usage-log:model/ApplicationPermissionsRevision metabase.permissions.models.application-permissions-revision@@ -126,6 +127,7 @@:model/RemoteSyncObject metabase-enterprise.remote-sync.models.remote-sync-object:model/RemoteSyncTask metabase-enterprise.remote-sync.models.remote-sync-task:model/ReplacementRun metabase-enterprise.replacement.models.replacement-run+ :model/ResultMetadataStatus metabase-enterprise.dependencies.models.result-metadata-status:model/Revision metabase.revisions.models.revision:model/SearchIndexMetadata metabase.search.models.search-index-metadata:model/Secret metabase.secrets.models.secretdiff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 2ee048f1ea7..3c4b71891f4 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1273,10 +1273,7 @@;; coerced to `{}`. See #74615.(m/update-existing :dataset_query #(lib-be/normalize-query nil % {:strict? true}))){:keys [metadata metadata-future]} (card.metadata/maybe-async-result-metadata- ;; 1. This function is called when storing metadata.- ;; 2. The metadata for storage shouldn't have remaps.- ;; 3. Setting this keyword will keep the remapped fields out of the metadata.- {:query (assoc-in (:dataset_query card-data) [:middleware :disable-remaps?] true)+ {:query (:dataset_query card-data):metadata result_metadata:entity-id (:entity_id card-data):model? (model? card-data)})@@ -1512,13 +1509,11 @@:result_metadata :collection_preview :verified-result-metadata?})](assert-is-valid-dashboard-internal-update card-updates card-before-update)(maybe-unverify! card-before-update updated-fields actor)- ;; Invalidate the cache for card- (cache/invalidate-config! {:questions [(:id card-before-update)]- :with-overrides? true})- ;; ok, now save the Card(queries.db/update-card! (:id card-before-update)(m/update-existing updated-fields :dataset_query- #(lib/normalize ::queries.schema/card.dataset-query %))))+ #(lib/normalize ::queries.schema/card.dataset-query %)))+ (cache/invalidate-config! {:questions [(:id card-before-update)]+ :with-overrides? true}));; Update all transitively dependent cards if the database was changed (#74561)(cascade-database-change-to-dependents! card-before-update card-updates);; ok, now update dependent dashcard parametersdiff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.cljindex 3f4ecf26fb4..ee3efc02ac7 100644--- a/src/metabase/queries/models/card/metadata.clj+++ b/src/metabase/queries/models/card/metadata.clj@@ -187,17 +187,11 @@ saved later when it is ready."(log/errorf "Error updating metadata for Card %d asynchronously: %s" id (ex-message e)))))))(defn infer-metadata- "Infer the default result_metadata to store for MBQL cards.-- Ignores any that might be present already."+ "Infer the default result_metadata to store for MBQL cards, as no user, ignoring any already present."[query](not-empty (request/with-current-user nil(u/ignore-exceptions- (qp.preprocess/query->expected-cols- ;; 1. This function is called when storing metadata.- ;; 2. The metadata for storage shouldn't have remaps.- ;; 3. Setting this keyword will keep the remapped fields out of the metadata.- (assoc-in query [:middleware :disable-remaps?] true))))))+ (qp.preprocess/query->expected-cols query)))))(defn infer-metadata-with-model-overrides"Does a fresh [[infer-metadata]] for the provided query.diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex e0a8dc37322..425b2b9ae96 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -639,6 +639,11 @@((map remap-fn) rf)rf))+(defn- cols-internal-columns-info+ "The [[internal-columns-info]] of the result columns `cols`."+ [cols]+ (internal-columns-info (map #(lib-be/instance->metadata % :metadata/column) cols)))+(mu/defn remap-results :- ::qp.schema/rff"Post-processing middleware. Handles `::external-remaps` added by [[add-remapped-columns-middleware]]; transformsresults and adds additional metadata based on these remaps, as well as internal (human-readable values) remaps."@@ -647,13 +652,19 @@(if disable-remaps?rff(fn remap-results-rff* [metadata]- (let [lib-cols (map- #(lib-be/instance->metadata % :metadata/column)- (:cols metadata))- internal-cols-info (internal-columns-info lib-cols)+ (let [internal-cols-info (cols-internal-columns-info (:cols metadata))metadata (add-remapped-to-and-from-metadata metadata external-remaps internal-cols-info)](remap-results-xform internal-cols-info (rff metadata))))))+(mu/defn remap-cols :- [:maybe ::qp.schema/result-metadata.columns]+ "The result columns `cols` of the preprocessed `query` with the remapping metadata [[remap-results]] adds to the+ results of running it."+ [{::keys [external-remaps], {:keys [disable-remaps?]} :middleware, :as _query} :- ::lib.schema/query+ cols :- [:maybe ::qp.schema/result-metadata.columns]]+ (if (or disable-remaps? (empty? cols))+ cols+ (vec (:cols (add-remapped-to-and-from-metadata {:cols cols} external-remaps (cols-internal-columns-info cols))))))+(defn disable-remaps"Sets the value of the disable-remaps? option in this query."[query]diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex c62b87566bd..1a96747dbb5 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -164,4 +164,4 @@(let [preprocessed (-> query preprocess)];; TODO - we should throw an Exception if the query has a native source query with no attached metadata or at;; least warn about it. Need to check where this is used.- (not-empty (annotate/expected-cols preprocessed)))))+ (not-empty (qp.add-remaps/remap-cols preprocessed (annotate/expected-cols preprocessed))))))diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex 0435c95b92d..ce1ed024adb 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -4,7 +4,6 @@(:require[buddy.core.codecs :as codecs][clojure.string :as str]- [medley.core :as m][metabase.driver :as driver][metabase.lib-be.core][metabase.lib.core :as lib]@@ -92,17 +91,11 @@[new-metadata old-metadata];; deprecated fn using its own deprecated helper; both leave together#_{:clj-kondo/ignore [:deprecated-var]}- (let [old-cols-by-desired-col-alias (m/index-by :lib/desired-column-alias (filter :lib/desired-column-alias old-metadata))- old-cols-by-name (m/index-by :name old-metadata)]- (for [new-col new-metadata]- (if-let [old-col (or (get old-cols-by-desired-col-alias (:lib/desired-column-alias new-col))- ;; Only match by name if a desired column alias is missing,- ;; otherwise we could match different columns with the same name- (let [old-col (get old-cols-by-name (:name new-col))]- (when (not-every? :lib/desired-column-alias [new-col old-col])- old-col)))]- (merge new-col (select-keys old-col preserved-keys))- new-col))))+ (map (fn [new-col old-col]+ (cond-> new-col+ old-col (merge (select-keys old-col preserved-keys))))+ new-metadata+ (lib/match-returned-columns old-metadata new-metadata)))#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:dynamic *execute-async?*diff --git a/src/metabase/sync/analyze/classify.clj b/src/metabase/sync/analyze/classify.cljindex 93d1408d07f..f0f1415f593 100644--- a/src/metabase/sync/analyze/classify.clj+++ b/src/metabase/sync/analyze/classify.clj@@ -19,6 +19,7 @@(:require[clojure.data :as data][metabase.analyze.core :as analyze]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.models.interface :as mi]@@ -118,16 +119,17 @@(when-let [fields (fields-to-classify table)](let [existing-name-field (sync.db/name-field-count-for-table table-id){:keys [fields-failed]}- (reduce (fn [state field]- (let [result (classify! field state)]- (cond-> state- (instance? Exception result)- (update :fields-failed inc)-- (= :type/Name (:semantic_type result))- (assoc :exists-name true))))- {:fields-failed 0 :exists-name (pos? existing-name-field)}- fields)]+ (dependencies.hooks/with-batched-marks+ (reduce (fn [state field]+ (let [result (classify! field state)]+ (cond-> state+ (instance? Exception result)+ (update :fields-failed inc)++ (= :type/Name (:semantic_type result))+ (assoc :exists-name true))))+ {:fields-failed 0 :exists-name (pos? existing-name-field)}+ fields))]{:fields-classified (count fields):fields-failed fields-failed})))))diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex f64a7a0b989..f1aa269e146 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -7,7 +7,6 @@`toucan2.core` itself."(:require[clojure.set :as set]- [honey.sql :as sql][metabase.app-db.core :as app-db][metabase.lib.schema.id :as lib.schema.id][metabase.util :as u]@@ -562,53 +561,6 @@[:= :t.active true][:= :t.visibility_type nil]]})-(defn- fk-target-changed-clause- "Honey SQL clause true when the Field's `fk_target_field_id` is not already `pk-id-expr`."- [pk-id-expr]- [:or- [:= :f.fk_target_field_id nil]- [:not= :f.fk_target_field_id pk-id-expr]])--(defn- mark-fk-statement- "`[sql & params]` updating the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name- fk-column-name]` in the Database with `db-id` to the id of the Field at `[pk-table-schema pk-table-name- pk-column-name]`, unless it already points there, per the application DB's dialect."- [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]- (let [fk-field-id-query (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)- pk-field-id-query (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)- q (case (app-db/db-type)- :mysql- {:update [(t2/table-name :model/Field) :f]- :join [[fk-field-id-query :fk] [:= :fk.id :f.id]- [pk-field-id-query :pk]- (fk-target-changed-clause :pk.id)]- :set {:fk_target_field_id :pk.id- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}}- :postgres- {:update [(t2/table-name :model/Field) :f]- :from [[fk-field-id-query :fk]]- :join [[pk-field-id-query :pk] true]- :set {:fk_target_field_id :pk.id- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}- :where [:and- [:= :fk.id :f.id]- (fk-target-changed-clause :pk.id)]}- :h2- {:update [(t2/table-name :model/Field) :f]- :set {:fk_target_field_id pk-field-id-query- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}- :where [:and- [:= :f.id fk-field-id-query]- [:not= pk-field-id-query nil]- (fk-target-changed-clause pk-field-id-query)]})]- (sql/format q :dialect (app-db/quoting-style (app-db/db-type)))))-(mu/defn mark-fk!"Set the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name fk-column-name]` in the Database with`db-id` to the id of the Field at `[pk-table-schema pk-table-name pk-column-name]`, unless it already points there.@@ -620,10 +572,15 @@pk-table-schema :- [:maybe :string]pk-table-name :- :stringpk-column-name :- :string]- (t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-name- pk-table-schema pk-table-name pk-column-name)))--;;; ------------------------------------------ Field data sensitivity ------------------------------------------+ (let [fk-id (:id (t2/query-one (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)))+ pk-id (:id (t2/query-one (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)))+ field (when (and fk-id pk-id)+ (t2/select-one [:model/Field :id :fk_target_field_id :has_field_values] :id fk-id+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))]+ (if (and field (not= pk-id (:fk_target_field_id field)))+ (t2/update! :model/Field fk-id (cond-> {:fk_target_field_id pk-id :semantic_type :type/FK}+ (= :auto-list (:has_field_values field)) (assoc :has_field_values nil)))+ 0)))(defn- data-sensitivity-to-scan-clause"Honey SQL clause matching Fields the data-sensitivity classifier still has to scan: unlabeled ones, plus those itdiff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.cljindex 49b4f4cfd27..1eabacb2d7a 100644--- a/src/metabase/sync/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata.clj@@ -8,7 +8,6 @@4. Sync FKs and indexes5. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"(:require- [metabase.dependencies.hooks :as dependencies.hooks][metabase.sync.fetch-metadata :as fetch-metadata][metabase.sync.interface :as i][metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]@@ -94,7 +93,6 @@(if abort-cause(sync-util/set-initial-database-sync-aborted! database abort-cause)(sync-util/set-initial-database-sync-complete! database))- (dependencies.hooks/after-database-sync (:id database))results))(mu/defn sync-db-metadata!diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.cljindex d7171834625..f172f394788 100644--- a/src/metabase/sync/sync_metadata/fields.clj+++ b/src/metabase/sync/sync_metadata/fields.clj@@ -39,6 +39,7 @@* In general the methods in these namespaces return the number of rows updated; these numbers are summed and usedfor logging purposes by higher-level sync logic."(:require+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.sync.db :as sync.db]@@ -94,11 +95,12 @@table :- i/TableInstancedb-metadata :- [:set i/TableMetadataField]](let [db-metadata (limit-fields-to-sync table db-metadata)]- (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))- ;; Now that tables are synced and fields created as needed make sure field properties are in sync.- ;; Re-fetch our metadata because there might be some things that have changed after calling- ;; `sync-instances`- (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table)))))+ (dependencies.hooks/with-batched-marks+ (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))+ ;; Now that tables are synced and fields created as needed make sure field properties are in sync.+ ;; Re-fetch our metadata because there might be some things that have changed after calling+ ;; `sync-instances`+ (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table))))))(defn- select-best-matching-name"Returns a key function for use with [[sort-by]] that ranks items based on how closely their `:schema` and `:name` match the given target values.diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.cljindex b5ce3088619..1f269f9b06c 100644--- a/src/metabase/sync/sync_metadata/tables.clj+++ b/src/metabase/sync/sync_metadata/tables.clj@@ -5,6 +5,7 @@[clojure.set :as set][java-time.api :as t][medley.core :as m]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.schema.common :as lib.schema.common]@@ -245,9 +246,10 @@(log/info "Found new table:"(sync-util/name-for-logging (mi/instance :model/Table table-metadata))))(let [field-order (some-> (:engine database) driver/default-field-order)]- (mapv (fn [table-metadata]- (create-table! database (m/assoc-some table-metadata :field_order field-order)))- (sort-by (juxt :schema :name) new-table-metadatas))))+ (dependencies.hooks/with-batched-marks+ (mapv (fn [table-metadata]+ (create-table! database (m/assoc-some table-metadata :field_order field-order)))+ (sort-by (juxt :schema :name) new-table-metadatas)))))(mu/defn- retire-tables!"Mark the active Tables with `table-ids` as inactive."diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.cljindex 2264255649c..8208e422f76 100644--- a/src/metabase/task/impl.clj+++ b/src/metabase/task/impl.clj@@ -160,6 +160,14 @@;;; | SCHEDULING/DELETING TASKS |;;; +----------------------------------------------------------------------------------------------------------------++(defn- schedule-trigger!+ "Schedule `trigger` for `job`, scheduling `job` with it when Quartz deleted the job since it was registered."+ [^Scheduler scheduler ^JobDetail job ^Trigger trigger]+ (try+ (.scheduleJob scheduler trigger)+ (catch JobPersistenceException _+ (.scheduleJob scheduler job trigger))))+(mu/defn- reschedule-task!"Assuming that [[job]] is already registered, ensure that [[new-trigger]] is scheduled to trigger it."[^Scheduler scheduler :- (ms/InstanceOfClass Scheduler)@@ -173,7 +181,7 @@replaced-trigger (or matching-trigger (first triggers))](log/debugf "Rescheduling job %s" (.getName job-key))(if-not replaced-trigger- (.scheduleJob scheduler new-trigger)+ (schedule-trigger! scheduler job new-trigger)(let [replaced-key (.getKey ^Trigger replaced-trigger)](when-not matching-trigger(log/warnf "Replacing trigger %s with trigger %s%s"@@ -184,7 +192,8 @@;; Ideally we would pass *all* the new triggers at once, so we can match them up atomically.;; The current behavior is especially confounding if replacing N triggers with M ones.(str " (chosen randomly from " (count triggers) " existing ones)"))))- (.rescheduleJob scheduler replaced-key new-trigger))))+ (when-not (.rescheduleJob scheduler replaced-key new-trigger)+ (schedule-trigger! scheduler job new-trigger)))))(catch Throwable e(log/errorf "Error rescheduling job: %s" (ex-message e)))))diff --git a/src/metabase/upload/db.clj b/src/metabase/upload/db.cljindex 257081b6d53..87070d268b9 100644--- a/src/metabase/upload/db.clj+++ b/src/metabase/upload/db.clj@@ -5,6 +5,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.queries.card-schema :as queries.card-schema][metabase.queries.schema :as queries.schema]+ [metabase.util :as u][metabase.util.malli :as mu][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][metabase.warehouse-schema.core :as warehouse-schema]@@ -62,20 +63,16 @@name->display-name :- [:map-of :string :string]](let [user-edited (user-edited-field-names table-id (set (keys name->display-name)))name->display-name (apply dissoc name->display-name user-edited)]- (when (seq name->display-name)- ;; A raw update rather than `t2/update!`, which produces an invalid query for certain versions of PostgreSQL:- ;; SELECT * FROM "metabase_field" WHERE "id" AND ("table_id" = ?) AND ... (argument of AND must be type boolean).- (t2/query {:update (t2/table-name :model/Field)- :set {:display_name (into [:case]- (mapcat identity)- (for [[n display-name] name->display-name]- [[:= [:lower :name] n]- [:case- [:= :display_name (humanization/name->human-readable-name n)] display-name- :else :display_name]]))}- :where [:and- [:= :table_id table-id]- [:in [:lower :name] (keys name->display-name)]]}))))+ (doseq [{:keys [id name display_name]} (when (seq name->display-name)+ (t2/select [:model/Field :id :name :display_name]+ {:from [(warehouse-schema-overlay/field-query+ {:user-settings? false})]+ :where [:and+ [:= :table_id table-id]+ [:in [:lower :name] (keys name->display-name)]]}))+ :let [lower-name (u/lower-case-en name)]+ :when (= display_name (humanization/name->human-readable-name lower-name))]+ (t2/update! :model/Field id {:display_name (name->display-name lower-name)}))))(mu/defn mark-table-upload!"Flag the Table with `table-id` as an authoritative, writable upload table."diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex edddb3969fd..c48f62a0c28 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -124,6 +124,7 @@(doto :model/Field(derive :metabase/model)(derive :hook/timestamped?)+ (derive :hook/dependencies);; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would;; cause duplication rather than good matching if the two instances are later linked by serdes.diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 3ebc5263184..ada9dc21d90 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -106,7 +106,8 @@(doto :model/FieldValues(derive :metabase/model)- (derive :hook/timestamped?))+ (derive :hook/timestamped?)+ (derive :hook/dependencies))(t2/deftransforms :model/FieldValues{:human_readable_values mi/transform-json-no-keywordizationdiff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 48c984bbd92..322d6b62397 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -94,6 +94,7 @@(derive ::mi/read-policy.full-perms-for-perms-set)(derive ::mi/write-policy.full-perms-for-perms-set)(derive :hook/timestamped?)+ (derive :hook/dependencies);; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would;; cause duplication rather than good matching if the two instances are later linked by serdes.@@ -255,13 +256,12 @@(cond;; active: true -> false (table being deactivated)(and (true? current-active) (false? new-active))- (assoc changes :deactivated_at (mi/now))+ (merge table changes {:deactivated_at (mi/now)});; active: false -> true (table being reactivated)(and (false? current-active) (true? new-active))- (assoc changes- :deactivated_at nil- :archived_at nil)+ (merge table changes {:deactivated_at nil+ :archived_at nil}):else (merge table changes)))))diff --git a/test/metabase/app_db/quartz_test.clj b/test/metabase/app_db/quartz_test.cljindex 7f0c0a618c4..17a89cdb748 100644--- a/test/metabase/app_db/quartz_test.clj+++ b/test/metabase/app_db/quartz_test.clj@@ -176,6 +176,7 @@"metabase_enterprise.data_complexity_score.task.complexity_score_trimmer.DataComplexityScoreTrimmer""metabase_enterprise.dependencies.task.backfill.BackfillDependencies""metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"+ "metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata""metabase_enterprise.entity_retrieval.task.sync.OsiAiContextSync""metabase_enterprise.mcp.task.mcp_usage_trimmer.McpUsageTrimmer""metabase_enterprise.metabot.task.ai_usage_trimmer.AiUsageTrimmer"diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex e13f89b63d4..8e287bc75bf 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -139,6 +139,7 @@:model/AiUsageLog:model/AnalysisFinding:model/AnalysisFindingError+ :model/AnalysisFindingStatus:model/ApiKeyUsageLog:model/CacheConfig:model/CardFavorite@@ -167,6 +168,7 @@:model/RemoteSyncObject:model/RemoteSyncTask:model/ReplacementRun+ :model/ResultMetadataStatus:model/SearchIndexMetadata:model/SecurityAdvisory:model/SemanticSearchTokenTrackingdiff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.cljindex 4625a19d909..79f749a0772 100644--- a/test/metabase/queries/models/card/metadata_test.clj+++ b/test/metabase/queries/models/card/metadata_test.clj@@ -5,6 +5,7 @@[metabase.lib.metadata :as lib.metadata][metabase.queries.models.card.metadata :as card.metadata][metabase.queries.schema :as queries.schema]+ [metabase.query-processor :as qp][metabase.test :as mt][toucan2.core :as t2]))@@ -48,8 +49,8 @@:base_type :type/BigInteger}]}(card.metadata/populate-result-metadata (lib/normalize ::queries.schema/card card)))))))-(deftest ^:parallel infer-metadata-no-remaps- (testing "infer-metadata should not include remapped columns (#67128)"+(deftest infer-metadata-remaps-test+ (testing "infer-metadata returns the remapped columns of a run, which a query on the card leaves out (#67128)"(mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :user_id):name "User ID":human_readable_field_id (mt/id :people :name)@@ -58,10 +59,26 @@:name "Product ID":human_readable_field_id (mt/id :products :title):type :external}]- (let [mp (mt/metadata-provider)- query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]- (is (= 9 ;; there are two remaps, giving 11 if included- (count (card.metadata/infer-metadata query))))))))+ (let [mp (mt/metadata-provider)+ query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ metadata (card.metadata/infer-metadata query)+ run (get-in (qp/process-query query) [:data :results_metadata :columns])+ remap (juxt :name :remapped_from :remapped_to)]+ (is (= (mapv remap run) (mapv remap metadata)))+ (is (= 2 (count (filter :remapped_from metadata))))+ (mt/with-temp [:model/Card {card-id :id} {:type :model :dataset_query query :result_metadata metadata}]+ (let [mp (mt/metadata-provider)]+ (is (= (mapv :name (lib/returned-columns query))+ (mapv :name (lib/returned-columns (lib/query mp (lib.metadata/card mp card-id))))))))))))++(deftest infer-metadata-internal-remaps-test+ (testing "infer-metadata returns the column of an internal remap that a run adds"+ (mt/with-column-remappings [venues.category_id (values-of categories.name)]+ (let [mp (mt/metadata-provider)+ query (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ remap (juxt :name :display_name :remapped_from :remapped_to :base_type)]+ (is (= (mapv remap (get-in (qp/process-query query) [:data :results_metadata :columns]))+ (mapv remap (card.metadata/infer-metadata query))))))))(deftest ^:parallel untyped-metadata?-test(testing "metadata is untyped when it has columns and none of them has a known base type (GHY-4213)"diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex d1caead3b34..0141c54504f 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -1836,7 +1836,7 @@(is (= :question(t2/select-one-fn :type :model/Card :id (:id card)))))))-(deftest create-card-no-remaps+(deftest create-card-remaps-test(mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Dimension _ {:field_id (mt/id :orders :user_id):name "User ID"@@ -1856,8 +1856,8 @@:dataset_query query}@api/*current-user*)](try- (is (= 9- (count (:result_metadata card))))+ (testing "a created card stores the remapped columns of a run"+ (is (= 2 (count (filter :remapped_from (:result_metadata card))))))(finally(t2/delete! :model/Card (:id card))))))))@@ -1911,9 +1911,9 @@:name "Orders Model for remap test":dataset_query query}@api/*current-user*)]- (testing "model result_metadata should not include remap columns"- (is (not-any? #(re-find #"(?i)remap" (:name %))- (:result_metadata card))))+ (testing "model result_metadata includes the remap column of a run, marked as remapped"+ (is (=? [{:remapped_from "QUANTITY"}]+ (filter #(re-find #"(?i)remap" (:name %)) (:result_metadata card)))))(let [mp (mt/metadata-provider)model-query (-> (lib/query mp (lib.metadata/card mp (:id card)))(lib/aggregate (lib/count))diff --git a/test/metabase/task_test.clj b/test/metabase/task_test.cljindex 04a6d721cb6..46ef792c60e 100644--- a/test/metabase/task_test.clj+++ b/test/metabase/task_test.clj@@ -103,6 +103,20 @@:misfire-instruction CronTrigger/MISFIRE_INSTRUCTION_IGNORE_MISFIRE_POLICY}}(triggers))))))+(deftest reschedule-after-job-was-deleted-test+ (testing "rescheduling schedules the new trigger when the job and its trigger are deleted while it reschedules, as+ Quartz does when the last run of a one-shot trigger completes"+ (with-temp-scheduler-and-cleanup!+ (task/schedule-task! (job) (trigger-1))+ (let [get-triggers (mt/original-fn #'qs/get-triggers-of-job)]+ (mt/with-dynamic-fn-redefs [qs/get-triggers-of-job (fn [scheduler job-key]+ (u/prog1 (get-triggers scheduler job-key)+ (qs/delete-job scheduler job-key)))]+ (task/schedule-task! (job) (trigger-2))))+ (is (= #{{:cron-expression "0 0 6 * * ? *"+ :misfire-instruction CronTrigger/MISFIRE_INSTRUCTION_IGNORE_MISFIRE_POLICY}}+ (triggers))))))+(deftest scheduler-info-test(testing "Make sure scheduler-info doesn't explode and returns info in the general shape we expect"(mt/with-temp-scheduler!