Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Run-stale-ancestors option for workspace transform execution (#69281)

Chris Truterfb97c31b6bc0
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
index 42a4be44979..4a1067dd6c0 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
@@ -39,6 +39,11 @@
(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)
+(defn- flag-enabled?
+ "Coerce a flag parameter (true, false, or 1) to boolean."
+ [v]
+ (boolean (#{1 true} v)))
+
(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])
(def ^:private computed-statuses
@@ -514,12 +519,12 @@
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
_query-params
;; Hmmm, I wonder why this isn't a boolean? T_T
- {:keys [stale_only]} :- [:map [:stale_only {:optional true} [:or [:= 1] :boolean]]]]
+ {:keys [stale_only]} :- [:map [:stale_only {:optional true} ::ws.t/flag]]]
(let [workspace (t2/select-one :model/Workspace :id ws-id)
_ (api/check-404 workspace)
_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
graph (ws.impl/get-or-calculate-graph! workspace)]
- (ws.impl/execute-workspace! workspace graph {:stale-only? (boolean (#{1 true} stale_only))})))
+ (ws.impl/execute-workspace! workspace graph {:stale-only? (flag-enabled? stale_only)})))
(mr/def ::graph-node-type [:enum :input-table :external-transform :workspace-transform])
@@ -915,30 +920,63 @@
:- ::ws.t/execution-result
"Run a transform in a workspace.
- App DB changes are rolled back. Warehouse DB changes persist."
+ App DB changes are rolled back. Warehouse DB changes persist.
+
+ When run_stale_ancestors is true, any stale ancestor transforms will be executed first
+ in dependency order. If any ancestor fails, execution stops and remaining ancestors
+ are marked as not_run. The target transform will not run if any ancestor failed."
{:access :workspace}
- [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
- (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
- _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
- _ (check-transforms-enabled! (:database_id workspace))
- graph (ws.impl/get-or-calculate-graph! workspace)]
- (ws.impl/run-transform! workspace graph transform)))
+ [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
+ _query-params
+ {:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]
+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
+ _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
+ _ (check-transforms-enabled! (:database_id workspace))
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ run-ancestors? (flag-enabled? run_stale_ancestors)
+ ancestors-result (when run-ancestors?
+ (ws.impl/run-stale-ancestors! workspace graph tx-id))
+ failed-ancestors (:failed ancestors-result)
+ transform-result (if (seq failed-ancestors)
+ {:status :failed
+ :message (str "Ancestor transform failed: "
+ (str/join ", " failed-ancestors))
+ :table (select-keys (:target transform) [:schema :name])}
+ (ws.impl/run-transform! workspace graph transform))]
+ (cond-> transform-result
+ run-ancestors? (assoc :ancestors ancestors-result))))
(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/dry-run"
:- ::ws.t/dry-run-result
"Dry-run a transform in a workspace without persisting to the target table.
Returns the first 2000 rows of transform output for preview purposes.
- Does not update last_run_at or create any database tables."
+ Does not update last_run_at or create any database tables.
+
+ When run_stale_ancestors is true, any stale ancestor transforms will be executed first
+ in dependency order (these ARE persisted). If any ancestor fails, execution stops and
+ the dry-run will not proceed."
{:access :workspace}
- [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
- (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
- _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
- _ (check-transforms-enabled! (:database_id workspace))
- graph (ws.impl/get-or-calculate-graph! workspace)]
- (ws.impl/dry-run-transform workspace graph transform)))
+ [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
+ _query-params
+ {:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]
+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
+ _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
+ _ (check-transforms-enabled! (:database_id workspace))
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ run-ancestors? (flag-enabled? run_stale_ancestors)
+ ancestors-result (when run-ancestors?
+ (ws.impl/run-stale-ancestors! workspace graph tx-id))
+ failed-ancestors (:failed ancestors-result)
+ dry-run-result (if (seq failed-ancestors)
+ {:status :failed
+ :message (str "Ancestor transform failed: "
+ (str/join ", " failed-ancestors))}
+ (ws.impl/dry-run-transform workspace graph transform))]
+ (cond-> dry-run-result
+ run-ancestors? (assoc :ancestors ancestors-result))))
(def ^:private CheckoutTransformLegacy
"Legacy format for workspace checkout transforms (DEPRECATED)."
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
index 827e5b2c5ca..ea9ff3ccf81 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
@@ -258,47 +258,78 @@
[]
(:t (first (t2/query {:select [[:%now :t]]}))))
-(def ^:private workspace-transform-id-xf
- "Transducer that extracts :id from workspace-transform nodes."
- (keep #(when (= :workspace-transform (:node-type %)) (:id %))))
+(defn- node-type-id-xf
+ "Create a transducer that filters nodes by type and extracts IDs."
+ [node-type]
+ (keep #(when (= node-type (:node-type %)) (:id %))))
+
+(defn- node-type-id-rf
+ "Create a reducing function that filters nodes by type and extracts IDs."
+ [node-type]
+ ((node-type-id-xf node-type) conj))
(defn- workspace-transform-ids
"Extract ref-ids from nodes that are workspace transforms."
[nodes]
- (into [] workspace-transform-id-xf nodes))
+ (into [] (node-type-id-xf :workspace-transform) nodes))
(declare get-or-calculate-graph!)
-(defn upstream-ancestors
- "Given a graph and a ref-id, find all upstream workspace transform ancestors.
- Returns ancestor ref-ids (not including the starting node).
+(defn upstream-nodes
+ "Given a graph and a workspace transform ref-id, find all upstream nodes (all node types).
+ Returns a set of {:node-type ... :id ...} maps (not including the starting node).
Takes a graph with :dependencies map (child->parents) and traverses upstream."
[graph ref-id]
(when-let [deps-map (:dependencies graph)]
- (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform :id ref-id}]
- :rf (workspace-transform-id-xf conj))))
+ (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform, :id ref-id}]
+ :init #{}
+ :rf (fn [acc entity]
+ (conj acc (select-keys entity [:node-type :id]))))))
+
+(defn upstream-ids
+ "Given a graph, filter node-type, and workspace transform ref-id, find upstream node IDs.
+ Returns IDs of upstream nodes matching the filter-type (not including the starting node).
+
+ Takes a graph with :dependencies map (child->parents) and traverses upstream."
+ [graph filter-type ref-id]
+ (when-let [deps-map (:dependencies graph)]
+ (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform, :id ref-id}]
+ :rf (node-type-id-rf filter-type))))
-(defn downstream-descendants
- "Given a graph and a ref-id, find all downstream workspace transform descendants.
- Returns descendant ref-ids (not including the starting node).
+(defn downstream-nodes
+ "Given a graph and a workspace transform ref-id, find all downstream nodes (all node types).
+ Returns a set of {:node-type ... :id ...} maps (not including the starting node).
Takes a graph with :dependencies map (child->parents), reverses it for downstream traversal."
[graph ref-id]
(when-let [deps-map (:dependencies graph)]
(let [forward-edges (ws.dag/reverse-graph deps-map)]
- (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform :id ref-id}]
- :rf (workspace-transform-id-xf conj)))))
+ (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform, :id ref-id}]
+ :init #{}
+ :rf (fn [acc entity]
+ (conj acc (select-keys entity [:node-type :id])))))))
+
+(defn downstream-ids
+ "Given a graph, filter node-type, and workspace transform ref-id, find downstream node IDs.
+ Returns IDs of downstream nodes matching the filter-type (not including the starting node).
+
+ Takes a graph with :dependencies map (child->parents), reverses it for downstream traversal."
+ [graph filter-type ref-id]
+ (when-let [deps-map (:dependencies graph)]
+ (let [forward-edges (ws.dag/reverse-graph deps-map)]
+ (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform, :id ref-id}]
+ :rf (node-type-id-rf filter-type)))))
(defn- any-internal-ancestor-stale?
"Check if any in-workspace entity ancestor is stale.
Returns true if any ancestor had its definition or input data changed since it last ran.
Note: external transforms are not checked here, their staleness must be checked separately."
[graph workspace ref-id]
- (when-let [upstream-ids (seq (upstream-ancestors graph ref-id))]
+ (when-let [ids (seq (upstream-ids graph :workspace-transform ref-id))]
(t2/exists? :model/WorkspaceTransform
:workspace_id (:id workspace)
- :ref_id [:in upstream-ids]
+ :ref_id [:in ids]
{:where [:or
[:= :definition_changed true]
[:= :input_data_changed true]]})))
@@ -307,10 +338,10 @@
"Mark all transitive downstream workspace transforms as input_data_changed.
Traverses the dependency graph downward from the given transform."
[graph workspace ref-id]
- (when-let [downstream-ids (seq (downstream-descendants graph ref-id))]
+ (when-let [ids (seq (downstream-ids graph :workspace-transform ref-id))]
(t2/update! :model/WorkspaceTransform
{:workspace_id (:id workspace)
- :ref_id [:in downstream-ids]}
+ :ref_id [:in ids]}
{:input_data_changed true})))
(defn run-transform!
@@ -731,28 +762,32 @@
(or (fully-persisted-graph ws-id graph-version)
(calculate-and-persist-graph! workspace graph-version)))
+(defn- entities->transforms
+ "Given a workspace-id and a sequence of graph entities, fetch the corresponding transform records.
+ Returns transforms in the same order as the input entities."
+ [ws-id entities]
+ (let [type->ids (u/group-by :node-type :id entities)
+ id->tx (merge
+ {}
+ (when-let [ids (seq (type->ids :external-transform))]
+ (t2/select-fn->fn :id identity
+ [:model/Transform :id :name :source :target]
+ :id [:in ids]))
+ (when-let [ref-ids (seq (type->ids :workspace-transform))]
+ (t2/select-fn->fn :ref_id identity
+ [:model/WorkspaceTransform :ref_id :name :source :target]
+ :workspace_id ws-id
+ :ref_id [:in ref-ids])))]
+ (keep (comp id->tx :id) entities)))
+
(defn- transforms-to-execute
"Given a workspace, graph, and an optional filter, return the global and workspace definitions to run, in order."
[{ws-id :id :as workspace} graph & {:keys [stale-only?]}]
- (let [graph (cond->> graph
- stale-only? (with-staleness workspace))
- entities (:entities graph)
- entities (if stale-only?
- (filter :stale entities)
- entities)
- type->ids (u/group-by :node-type :id entities)
- id->tx (merge
- {}
- (when-let [ids (seq (type->ids :external-transform))]
- (t2/select-fn->fn :id identity
- [:model/Transform :id :name :source :target]
- :id [:in ids]))
- (when-let [ref-ids (seq (type->ids :workspace-transform))]
- (t2/select-fn->fn :ref_id identity
- [:model/WorkspaceTransform :ref_id :name :source :target]
- :workspace_id ws-id
- :ref_id [:in ref-ids])))]
- (keep (comp id->tx :id) entities)))
+ (let [graph (cond->> graph
+ stale-only? (with-staleness workspace))
+ entities (cond->> (:entities graph)
+ stale-only? (filter :stale))]
+ (entities->transforms ws-id entities)))
(defn- id->str [ref-id-or-id]
(if (string? ref-id-or-id)
@@ -781,3 +816,59 @@
:failed []
:not_run []}
(transforms-to-execute workspace graph {:stale-only? stale-only?}))))
+
+(defn- stale-ancestors-to-execute
+ "Given a workspace, graph, and target ref-id, return the stale ancestor transforms to run, in dependency order.
+ Only returns transforms that are both ancestors of the target AND stale.
+ Includes both workspace transforms and external transforms in the ancestor chain."
+ [{ws-id :id} graph ref-id]
+ (let [ancestor-keys (upstream-nodes graph ref-id)
+ ancestor? (fn [entity] (contains? ancestor-keys (select-keys entity [:node-type :id])))
+ ancestor-entities (filter ancestor? (:entities graph))
+ ;; Build a subgraph with only ancestor entities and their inter-dependencies
+ ;; Dependencies map is keyed by entities (maps with :id), not by IDs directly
+ ancestor-deps (into {}
+ (keep (fn [[to-entity from-list]]
+ (when (ancestor? to-entity)
+ [to-entity (filterv ancestor? from-list)]))
+ (:dependencies graph)))
+ ancestor-graph {:entities ancestor-entities
+ :dependencies ancestor-deps}
+ ;; Only fetch staleness for ancestors, not the entire graph
+ ancestor-ref-ids (workspace-transform-ids ancestor-entities)
+ staleness-map (fetch-staleness-map ws-id ancestor-ref-ids)
+ annotated-graph (annotate-staleness ancestor-graph staleness-map)
+ stale-entities (filter :stale (:entities annotated-graph))]
+ (entities->transforms ws-id stale-entities)))
+
+(defn run-stale-ancestors!
+ "Run all stale ancestors of a given transform in dependency order.
+ Stops on first failure, marking remaining transforms as not_run.
+ Returns a map with :succeeded, :failed, and :not_run lists of ref-ids/transform-ids."
+ [workspace graph ref-id]
+ (let [ws-id (:id workspace)
+ remapping (build-remapping workspace graph)]
+ (reduce
+ (fn [acc {external-id :id tx-ref-id :ref_id :as transform}]
+ (let [node-type (if external-id :external-transform :workspace-transform)
+ id-str (id->str (or external-id tx-ref-id))]
+ (if (seq (:failed acc))
+ ;; Skip execution if any previous transform failed
+ (update acc :not_run conj id-str)
+ (let [result (try
+ (run-transform! workspace graph transform remapping)
+ (catch Exception e
+ {:status :failed
+ :message (ex-message e)
+ :error e}))]
+ (if (= :succeeded (:status result))
+ (update acc :succeeded conj id-str)
+ (let [log-data {:workspace-id ws-id :node-type node-type :id id-str :result (dissoc result :error)}]
+ (if-let [e (:error result)]
+ (log/error e "Failed to execute ancestor transform" log-data)
+ (log/error "Failed to execute ancestor transform" log-data))
+ (update acc :failed conj id-str)))))))
+ {:succeeded []
+ :failed []
+ :not_run []}
+ (stale-ancestors-to-execute workspace graph ref-id))))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
index a8408ce2c49..6ed2e5e69cf 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
@@ -7,6 +7,10 @@
(mr/def ::ref-id [:string {:min 1 :api/regex #".+"}])
+(mr/def ::flag
+ "Boolean flag that accepts true, false, or 1 (for query param compatibility)."
+ [:or [:= 1] :boolean])
+
(def entity-types
"The kinds of entities we can store within a Workspace."
[{:name "Transform"
@@ -22,6 +26,13 @@
(mr/def ::entity-map
[:map-of ::entity-grouping [:sequential ms/PositiveInt]])
+(mr/def ::ancestors-result
+ "Result of running stale ancestor transforms."
+ [:map
+ [:succeeded [:sequential ::ref-id]]
+ [:failed [:sequential ::ref-id]]
+ [:not_run [:sequential ::ref-id]]])
+
(mr/def ::execution-result
[:map
[:status [:enum :succeeded :failed]]
@@ -30,7 +41,8 @@
[:message {:optional true} [:maybe :string]]
[:table [:map
[:name :string]
- [:schema {:optional true} [:maybe :string]]]]])
+ [:schema {:optional true} [:maybe :string]]]]
+ [:ancestors {:optional true} ::ancestors-result]])
(mr/def ::dry-run-result
"Result of a transform dry-run (preview without persisting).
@@ -43,7 +55,8 @@
[:rows {:optional true} [:sequential :any]]
[:cols {:optional true} [:sequential :map]]
[:results_metadata {:optional true} [:map
- [:columns {:optional true} [:sequential :map]]]]]]])
+ [:columns {:optional true} [:sequential :map]]]]]]
+ [:ancestors {:optional true} ::ancestors-result]])
;;; ---------------------------------------- Graph/Problem Types ----------------------------------------
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index 2d5c792ae65..215a22349a8 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -5,6 +5,7 @@
[java-time.api :as t]
[medley.core :as m]
[metabase-enterprise.workspaces.common :as ws.common]
+ [metabase-enterprise.workspaces.execute :as ws.execute]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.test-util :as ws.tu]
@@ -23,7 +24,9 @@
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
[metabase.util :as u]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.time Instant)))
(set! *warn-on-reflection* true)
@@ -1299,6 +1302,74 @@
:message string?}
(mt/user-http-request :crowberto :post 200 (ws-url (:id ws1) "transform" ref-id "dry-run"))))))))))
+(deftest run-transform-with-stale-ancestors-test
+ (testing "POST /api/ee/workspace/:id/transform/:txid/run with run_stale_ancestors=true"
+ ;; Chain: x1 -> x2 -> x3, where x1 is stale
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}
+ :properties {:x1 {:definition_changed true}
+ :x2 {:definition_changed false}
+ :x3 {:definition_changed false}}}}]
+ (ws.tu/ws-done! workspace-id)
+ (let [x1-ref (workspace-map :x1)
+ x2-ref (workspace-map :x2)
+ x3-ref (workspace-map :x3)]
+ ;; Use a mock that returns :table (required by the API schema)
+ (mt/with-dynamic-fn-redefs [ws.execute/run-transform-with-remapping
+ (fn [{:keys [target]} _remapping]
+ {:status :succeeded
+ :end_time (Instant/now)
+ :message "Mocked execution"
+ :table (select-keys target [:schema :name])})]
+ (testing "without flag, ancestors are not run"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url workspace-id "/transform/" x3-ref "/run"))]
+ (is (= "succeeded" (:status result)))
+ (is (nil? (:ancestors result)))))
+ (testing "with flag, stale ancestors are run before target"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url workspace-id "/transform/" x3-ref "/run")
+ {:run_stale_ancestors true})]
+ (is (= "succeeded" (:status result)))
+ (is (= {:succeeded [x1-ref x2-ref]
+ :failed []
+ :not_run []}
+ (:ancestors result))))))))))
+
+(deftest dry-run-transform-with-stale-ancestors-test
+ (testing "POST /api/ee/workspace/:id/transform/:txid/dry-run with run_stale_ancestors=true"
+ ;; Chain: x1 -> x2 -> x3, where x1 is stale
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}
+ :properties {:x1 {:definition_changed true}
+ :x2 {:definition_changed false}
+ :x3 {:definition_changed false}}}}]
+ (ws.tu/ws-done! workspace-id)
+ (let [x1-ref (workspace-map :x1)
+ x2-ref (workspace-map :x2)
+ x3-ref (workspace-map :x3)]
+ ;; Mock both run (for ancestors) and preview (for dry-run target)
+ (mt/with-dynamic-fn-redefs [ws.execute/run-transform-with-remapping
+ (fn [{:keys [target]} _remapping]
+ {:status :succeeded
+ :end_time (Instant/now)
+ :message "Mocked execution"
+ :table (select-keys target [:schema :name])})
+ ws.execute/run-transform-preview
+ (fn [_transform _remapping]
+ {:status :succeeded
+ :data {:rows [[1 "test"]]
+ :cols [{:name "id"} {:name "name"}]}})]
+ (testing "with flag, stale ancestors are run before dry-run"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url workspace-id "/transform/" x3-ref "/dry-run")
+ {:run_stale_ancestors true})]
+ (is (= "succeeded" (:status result)))
+ (is (= {:succeeded [x1-ref x2-ref]
+ :failed []
+ :not_run []}
+ (:ancestors result))))))))))
+
(defn- random-target [db-id]
{:type "table"
:database db-id
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj
index e78ca914381..c35891396b8 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj
@@ -358,9 +358,9 @@
(is (true? (:definition_changed entity)))
(is (false? (:input_data_changed entity))))))
-;;;; Static graph tests for marking functions (upstream-ancestors, downstream-descendants, any-ancestor-stale?)
+;;;; Static graph tests for graph traversal functions
-(deftest upstream-ancestors-linear-chain-test
+(deftest upstream-ids-linear-chain-test
(testing "Linear chain: computes all upstream ancestors"
(let [t1 (workspace-transform "t1")
t2 (workspace-transform "t2")
@@ -371,14 +371,14 @@
t3 [t2]}}]
(testing "t3 has t1 and t2 as ancestors"
(is (= #{"t1" "t2"}
- (set (ws.impl/upstream-ancestors graph "t3")))))
+ (set (ws.impl/upstream-ids graph :workspace-transform "t3")))))
(testing "t2 has only t1 as ancestor"
(is (= #{"t1"}
- (set (ws.impl/upstream-ancestors graph "t2")))))
+ (set (ws.impl/upstream-ids graph :workspace-transform "t2")))))
(testing "t1 has no ancestors"
- (is (empty? (ws.impl/upstream-ancestors graph "t1")))))))
+ (is (empty? (ws.impl/upstream-ids graph :workspace-transform "t1")))))))
-(deftest upstream-ancestors-diamond-test
+(deftest upstream-ids-diamond-test
(testing "Diamond graph: computes all paths"
(let [t1 (workspace-transform "t1")
t2 (workspace-transform "t2")
@@ -395,13 +395,13 @@
t4 [t2 t3]}}]
(testing "t4 has all transforms as ancestors"
(is (= #{"t1" "t2" "t3"}
- (set (ws.impl/upstream-ancestors graph "t4")))))
+ (set (ws.impl/upstream-ids graph :workspace-transform "t4")))))
(testing "t2 has only t1 as ancestor"
(is (= #{"t1"}
- (set (ws.impl/upstream-ancestors graph "t2"))))))))
+ (set (ws.impl/upstream-ids graph :workspace-transform "t2"))))))))
-(deftest upstream-ancestors-with-external-transforms-test
- (testing "External transforms are traversed but not included"
+(deftest upstream-ids-with-external-transforms-test
+ (testing "External transforms are traversed but not included when filtered"
(let [t1 (workspace-transform "t1")
ext (global-transform 100)
t2 (workspace-transform "t2")
@@ -411,11 +411,13 @@
t2 [ext]}}]
(testing "t2 can reach through external transform to t1"
(is (= #{"t1"}
- (set (ws.impl/upstream-ancestors graph "t2")))))
- (testing "ext cannot be returned as it's external"
- (is (not (contains? (set (ws.impl/upstream-ancestors graph "t2")) 100)))))))
+ (set (ws.impl/upstream-ids graph :workspace-transform "t2")))))
+ (testing "upstream-nodes returns all node types including external"
+ (is (= #{{:node-type :workspace-transform :id "t1"}
+ {:node-type :external-transform :id 100}}
+ (ws.impl/upstream-nodes graph "t2")))))))
-(deftest downstream-descendants-linear-chain-test
+(deftest downstream-ids-linear-chain-test
(testing "Linear chain: computes all downstream descendants"
(let [t1 (workspace-transform "t1")
t2 (workspace-transform "t2")
@@ -426,14 +428,14 @@
t3 [t2]}}]
(testing "t1 has t2 and t3 as descendants"
(is (= #{"t2" "t3"}
- (set (ws.impl/downstream-descendants graph "t1")))))
+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))
(testing "t2 has only t3 as descendant"
(is (= #{"t3"}
- (set (ws.impl/downstream-descendants graph "t2")))))
+ (set (ws.impl/downstream-ids graph :workspace-transform "t2")))))
(testing "t3 has no descendants"
- (is (empty? (ws.impl/downstream-descendants graph "t3")))))))
+ (is (empty? (ws.impl/downstream-ids graph :workspace-transform "t3")))))))
-(deftest downstream-descendants-diamond-test
+(deftest downstream-ids-diamond-test
(testing "Diamond graph: computes all downstream paths"
(let [t1 (workspace-transform "t1")
t2 (workspace-transform "t2")
@@ -450,13 +452,13 @@
t4 [t2 t3]}}]
(testing "t1 has all transforms as descendants"
(is (= #{"t2" "t3" "t4"}
- (set (ws.impl/downstream-descendants graph "t1")))))
+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))
(testing "t2 has only t4 as descendant"
(is (= #{"t4"}
- (set (ws.impl/downstream-descendants graph "t2"))))))))
+ (set (ws.impl/downstream-ids graph :workspace-transform "t2"))))))))
-(deftest downstream-descendants-with-external-transforms-test
- (testing "External transforms are traversed but not included"
+(deftest downstream-ids-with-external-transforms-test
+ (testing "External transforms are traversed but not included when filtered"
(let [t1 (workspace-transform "t1")
ext (global-transform 100)
t2 (workspace-transform "t2")
@@ -466,14 +468,143 @@
t2 [ext]}}]
(testing "t1 can reach through external transform to t2"
(is (= #{"t2"}
- (set (ws.impl/downstream-descendants graph "t1")))))
- (testing "ext cannot be returned as it's external"
- (is (not (contains? (set (ws.impl/downstream-descendants graph "t1")) 100)))))))
+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))
+ (testing "downstream-nodes returns all node types including external"
+ (is (= #{{:node-type :workspace-transform, :id "t2"}
+ {:node-type :external-transform, :id 100}}
+ (ws.impl/downstream-nodes graph "t1")))))))
-(deftest downstream-descendants-empty-graph-test
+(deftest downstream-nodes-empty-graph-test
(testing "Empty graph returns nil"
- (is (nil? (ws.impl/downstream-descendants {:entities [] :dependencies nil} "t1")))))
+ (is (nil? (ws.impl/downstream-nodes {:entities [] :dependencies nil} "t1")))))
-(deftest upstream-ancestors-empty-graph-test
+(deftest upstream-nodes-empty-graph-test
(testing "Empty graph returns nil"
- (is (nil? (ws.impl/upstream-ancestors {:entities [] :dependencies nil} "t1")))))
+ (is (nil? (ws.impl/upstream-nodes {:entities [] :dependencies nil} "t1")))))
+
+;;;; run-stale-ancestors! tests
+
+(deftest run-stale-ancestors-runs-only-stale-ancestors-test
+ (testing "run-stale-ancestors! only runs ancestors that are stale (including transitively stale)"
+ ;; Graph:
+ ;; x1 -> x2 \
+ ;; x3 -> x4 -> x7
+ ;; x5 -> x6 /
+ ;; x1 and x4 are locally stale, x2 is transitively stale (depends on x1)
+ ;; x3, x5, x6 are fresh and should NOT run
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1]
+ :x3 [:t0] :x4 [:x3]
+ :x5 [:t0] :x6 [:x5]
+ :x7 [:x2 :x4 :x6]}
+ :properties {:x1 {:definition_changed true}
+ :x2 {:definition_changed false}
+ :x3 {:definition_changed false}
+ :x4 {:definition_changed true}
+ :x5 {:definition_changed false}
+ :x6 {:definition_changed false}
+ :x7 {:definition_changed false}}}}]
+ (let [t1-ref (workspace-map :x1)
+ t2-ref (workspace-map :x2)
+ t4-ref (workspace-map :x4)
+ t7-ref (workspace-map :x7)]
+ (ws.tu/with-mocked-execution
+ (let [workspace (t2/select-one :model/Workspace workspace-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ result (ws.impl/run-stale-ancestors! workspace graph t7-ref)
+ succeeded (:succeeded result)]
+ (testing "x1, x2 (transitively stale), and x4 run; x3, x5, x6 do not"
+ (is (= (set [t1-ref t2-ref t4-ref]) (set succeeded)))
+ (is (= [] (:failed result)))
+ (is (= [] (:not_run result))))
+ (testing "x1 runs before x2 (dependency order within branch)"
+ (is (< (.indexOf succeeded t1-ref) (.indexOf succeeded t2-ref))))))))))
+
+(deftest run-stale-ancestors-respects-dependency-order-test
+ (testing "run-stale-ancestors! runs ancestors in dependency order"
+ ;; Chain: t1 -> t2 -> t3, where t1 and t2 are stale
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}
+ :properties {:x1 {:definition_changed true}
+ :x2 {:definition_changed true}
+ :x3 {:definition_changed false}}}}]
+ (let [t1-ref (workspace-map :x1)
+ t2-ref (workspace-map :x2)
+ t3-ref (workspace-map :x3)
+ run-order (atom [])]
+ (mt/with-dynamic-fn-redefs [ws.impl/run-transform!
+ (fn [_ws _graph transform & _]
+ (swap! run-order conj (:ref_id transform))
+ {:status :succeeded})]
+ (let [workspace (t2/select-one :model/Workspace workspace-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ result (ws.impl/run-stale-ancestors! workspace graph t3-ref)]
+ (testing "both stale ancestors are run in order (t1 before t2)"
+ (is (= [t1-ref t2-ref] @run-order))
+ (is (= [t1-ref t2-ref] (:succeeded result))))))))))
+
+(deftest run-stale-ancestors-stops-on-failure-test
+ (testing "run-stale-ancestors! stops on first failure and marks remaining as not_run"
+ ;; Chain: t1 -> t2 -> t3, where t1 and t2 are stale, t1 fails
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}
+ :properties {:x1 {:definition_changed true}
+ :x2 {:definition_changed true}
+ :x3 {:definition_changed false}}}}]
+ (let [t1-ref (workspace-map :x1)
+ t2-ref (workspace-map :x2)
+ t3-ref (workspace-map :x3)]
+ (mt/with-dynamic-fn-redefs [ws.impl/run-transform!
+ (fn [_ws _graph transform & _]
+ (if (= (:ref_id transform) t1-ref)
+ {:status :failed :message "Simulated failure"}
+ {:status :succeeded}))]
+ (let [workspace (t2/select-one :model/Workspace workspace-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ result (ws.impl/run-stale-ancestors! workspace graph t3-ref)]
+ (testing "t1 fails, t2 is not_run"
+ (is (= [] (:succeeded result)))
+ (is (= [t1-ref] (:failed result)))
+ (is (= [t2-ref] (:not_run result))))))))))
+
+(deftest run-stale-ancestors-no-stale-ancestors-test
+ (testing "run-stale-ancestors! returns empty results when no ancestors are stale"
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1]}
+ :properties {:x1 {:definition_changed false}
+ :x2 {:definition_changed false}}}}]
+ (let [t2-ref (workspace-map :x2)]
+ (ws.tu/with-mocked-execution
+ (let [workspace (t2/select-one :model/Workspace workspace-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ result (ws.impl/run-stale-ancestors! workspace graph t2-ref)]
+ (testing "no transforms are run"
+ (is (= [] (:succeeded result)))
+ (is (= [] (:failed result)))
+ (is (= [] (:not_run result))))))))))
+
+(deftest run-stale-ancestors-includes-external-transforms-test
+ (testing "run-stale-ancestors! includes enclosed external transforms in the ancestor chain"
+ ;; Graph: (x1) -> x2 -> (x3) where x2 is an external (global) transform
+ ;; x1 is a workspace checkout, x2 is enclosed (global, not checked out), x3 is a workspace checkout
+ ;; x1 is locally stale -> x2 becomes transitively stale -> running ancestors for x3 should run both x1 and x2
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map global-map]}
+ {:global {:x1 [:t0] :x2 [:x1] :x3 [:x2]}
+ :workspace {:checkouts [:x1 :x3]
+ :properties {:x1 {:definition_changed true}
+ :x3 {:definition_changed false}}}}]
+ (ws.tu/ws-done! workspace-id)
+ (let [x1-ref (workspace-map :x1)
+ x2-global (str "global-id:" (global-map :x2))
+ x3-ref (workspace-map :x3)]
+ (ws.tu/with-mocked-execution
+ (let [workspace (t2/select-one :model/Workspace workspace-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)
+ result (ws.impl/run-stale-ancestors! workspace graph x3-ref)
+ succeeded (:succeeded result)]
+ (testing "both workspace transform x1 and external transform x2 are run"
+ (is (= #{x1-ref x2-global} (set succeeded)))
+ (is (= [] (:failed result)))
+ (is (= [] (:not_run result))))
+ (testing "x1 runs before x2 (dependency order)"
+ (is (< (.indexOf succeeded x1-ref) (.indexOf succeeded x2-global))))))))))