The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Run-stale-ancestors option for workspace transform execution (#69281)
Chris Truterfb97c31b6bc0
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex 42a4be44979..4a1067dd6c0 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -39,6 +39,11 @@(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)+(defn- flag-enabled?+ "Coerce a flag parameter (true, false, or 1) to boolean."+ [v]+ (boolean (#{1 true} v)))+(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])(def ^:private computed-statuses@@ -514,12 +519,12 @@[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]_query-params;; Hmmm, I wonder why this isn't a boolean? T_T- {:keys [stale_only]} :- [:map [:stale_only {:optional true} [:or [:= 1] :boolean]]]]+ {:keys [stale_only]} :- [:map [:stale_only {:optional true} ::ws.t/flag]]](let [workspace (t2/select-one :model/Workspace :id ws-id)_ (api/check-404 workspace)_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")graph (ws.impl/get-or-calculate-graph! workspace)]- (ws.impl/execute-workspace! workspace graph {:stale-only? (boolean (#{1 true} stale_only))})))+ (ws.impl/execute-workspace! workspace graph {:stale-only? (flag-enabled? stale_only)})))(mr/def ::graph-node-type [:enum :input-table :external-transform :workspace-transform])@@ -915,30 +920,63 @@:- ::ws.t/execution-result"Run a transform in a workspace.- App DB changes are rolled back. Warehouse DB changes persist."+ App DB changes are rolled back. Warehouse DB changes persist.++ When run_stale_ancestors is true, any stale ancestor transforms will be executed first+ in dependency order. If any ancestor fails, execution stops and remaining ancestors+ are marked as not_run. The target transform will not run if any ancestor failed."{:access :workspace}- [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]- (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))- _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")- _ (check-transforms-enabled! (:database_id workspace))- graph (ws.impl/get-or-calculate-graph! workspace)]- (ws.impl/run-transform! workspace graph transform)))+ [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]+ _query-params+ {:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))+ _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")+ _ (check-transforms-enabled! (:database_id workspace))+ graph (ws.impl/get-or-calculate-graph! workspace)+ run-ancestors? (flag-enabled? run_stale_ancestors)+ ancestors-result (when run-ancestors?+ (ws.impl/run-stale-ancestors! workspace graph tx-id))+ failed-ancestors (:failed ancestors-result)+ transform-result (if (seq failed-ancestors)+ {:status :failed+ :message (str "Ancestor transform failed: "+ (str/join ", " failed-ancestors))+ :table (select-keys (:target transform) [:schema :name])}+ (ws.impl/run-transform! workspace graph transform))]+ (cond-> transform-result+ run-ancestors? (assoc :ancestors ancestors-result))))(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/dry-run":- ::ws.t/dry-run-result"Dry-run a transform in a workspace without persisting to the target table.Returns the first 2000 rows of transform output for preview purposes.- Does not update last_run_at or create any database tables."+ Does not update last_run_at or create any database tables.++ When run_stale_ancestors is true, any stale ancestor transforms will be executed first+ in dependency order (these ARE persisted). If any ancestor fails, execution stops and+ the dry-run will not proceed."{:access :workspace}- [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]- (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))- _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")- _ (check-transforms-enabled! (:database_id workspace))- graph (ws.impl/get-or-calculate-graph! workspace)]- (ws.impl/dry-run-transform workspace graph transform)))+ [{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]+ _query-params+ {:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))+ _ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")+ _ (check-transforms-enabled! (:database_id workspace))+ graph (ws.impl/get-or-calculate-graph! workspace)+ run-ancestors? (flag-enabled? run_stale_ancestors)+ ancestors-result (when run-ancestors?+ (ws.impl/run-stale-ancestors! workspace graph tx-id))+ failed-ancestors (:failed ancestors-result)+ dry-run-result (if (seq failed-ancestors)+ {:status :failed+ :message (str "Ancestor transform failed: "+ (str/join ", " failed-ancestors))}+ (ws.impl/dry-run-transform workspace graph transform))]+ (cond-> dry-run-result+ run-ancestors? (assoc :ancestors ancestors-result))))(def ^:private CheckoutTransformLegacy"Legacy format for workspace checkout transforms (DEPRECATED)."diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.cljindex 827e5b2c5ca..ea9ff3ccf81 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj@@ -258,47 +258,78 @@[](:t (first (t2/query {:select [[:%now :t]]}))))-(def ^:private workspace-transform-id-xf- "Transducer that extracts :id from workspace-transform nodes."- (keep #(when (= :workspace-transform (:node-type %)) (:id %))))+(defn- node-type-id-xf+ "Create a transducer that filters nodes by type and extracts IDs."+ [node-type]+ (keep #(when (= node-type (:node-type %)) (:id %))))++(defn- node-type-id-rf+ "Create a reducing function that filters nodes by type and extracts IDs."+ [node-type]+ ((node-type-id-xf node-type) conj))(defn- workspace-transform-ids"Extract ref-ids from nodes that are workspace transforms."[nodes]- (into [] workspace-transform-id-xf nodes))+ (into [] (node-type-id-xf :workspace-transform) nodes))(declare get-or-calculate-graph!)-(defn upstream-ancestors- "Given a graph and a ref-id, find all upstream workspace transform ancestors.- Returns ancestor ref-ids (not including the starting node).+(defn upstream-nodes+ "Given a graph and a workspace transform ref-id, find all upstream nodes (all node types).+ Returns a set of {:node-type ... :id ...} maps (not including the starting node).Takes a graph with :dependencies map (child->parents) and traverses upstream."[graph ref-id](when-let [deps-map (:dependencies graph)]- (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform :id ref-id}]- :rf (workspace-transform-id-xf conj))))+ (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform, :id ref-id}]+ :init #{}+ :rf (fn [acc entity]+ (conj acc (select-keys entity [:node-type :id]))))))++(defn upstream-ids+ "Given a graph, filter node-type, and workspace transform ref-id, find upstream node IDs.+ Returns IDs of upstream nodes matching the filter-type (not including the starting node).++ Takes a graph with :dependencies map (child->parents) and traverses upstream."+ [graph filter-type ref-id]+ (when-let [deps-map (:dependencies graph)]+ (ws.dag/bfs-reduce deps-map [{:node-type :workspace-transform, :id ref-id}]+ :rf (node-type-id-rf filter-type))))-(defn downstream-descendants- "Given a graph and a ref-id, find all downstream workspace transform descendants.- Returns descendant ref-ids (not including the starting node).+(defn downstream-nodes+ "Given a graph and a workspace transform ref-id, find all downstream nodes (all node types).+ Returns a set of {:node-type ... :id ...} maps (not including the starting node).Takes a graph with :dependencies map (child->parents), reverses it for downstream traversal."[graph ref-id](when-let [deps-map (:dependencies graph)](let [forward-edges (ws.dag/reverse-graph deps-map)]- (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform :id ref-id}]- :rf (workspace-transform-id-xf conj)))))+ (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform, :id ref-id}]+ :init #{}+ :rf (fn [acc entity]+ (conj acc (select-keys entity [:node-type :id])))))))++(defn downstream-ids+ "Given a graph, filter node-type, and workspace transform ref-id, find downstream node IDs.+ Returns IDs of downstream nodes matching the filter-type (not including the starting node).++ Takes a graph with :dependencies map (child->parents), reverses it for downstream traversal."+ [graph filter-type ref-id]+ (when-let [deps-map (:dependencies graph)]+ (let [forward-edges (ws.dag/reverse-graph deps-map)]+ (ws.dag/bfs-reduce forward-edges [{:node-type :workspace-transform, :id ref-id}]+ :rf (node-type-id-rf filter-type)))))(defn- any-internal-ancestor-stale?"Check if any in-workspace entity ancestor is stale.Returns true if any ancestor had its definition or input data changed since it last ran.Note: external transforms are not checked here, their staleness must be checked separately."[graph workspace ref-id]- (when-let [upstream-ids (seq (upstream-ancestors graph ref-id))]+ (when-let [ids (seq (upstream-ids graph :workspace-transform ref-id))](t2/exists? :model/WorkspaceTransform:workspace_id (:id workspace)- :ref_id [:in upstream-ids]+ :ref_id [:in ids]{:where [:or[:= :definition_changed true][:= :input_data_changed true]]})))@@ -307,10 +338,10 @@"Mark all transitive downstream workspace transforms as input_data_changed.Traverses the dependency graph downward from the given transform."[graph workspace ref-id]- (when-let [downstream-ids (seq (downstream-descendants graph ref-id))]+ (when-let [ids (seq (downstream-ids graph :workspace-transform ref-id))](t2/update! :model/WorkspaceTransform{:workspace_id (:id workspace)- :ref_id [:in downstream-ids]}+ :ref_id [:in ids]}{:input_data_changed true})))(defn run-transform!@@ -731,28 +762,32 @@(or (fully-persisted-graph ws-id graph-version)(calculate-and-persist-graph! workspace graph-version)))+(defn- entities->transforms+ "Given a workspace-id and a sequence of graph entities, fetch the corresponding transform records.+ Returns transforms in the same order as the input entities."+ [ws-id entities]+ (let [type->ids (u/group-by :node-type :id entities)+ id->tx (merge+ {}+ (when-let [ids (seq (type->ids :external-transform))]+ (t2/select-fn->fn :id identity+ [:model/Transform :id :name :source :target]+ :id [:in ids]))+ (when-let [ref-ids (seq (type->ids :workspace-transform))]+ (t2/select-fn->fn :ref_id identity+ [:model/WorkspaceTransform :ref_id :name :source :target]+ :workspace_id ws-id+ :ref_id [:in ref-ids])))]+ (keep (comp id->tx :id) entities)))+(defn- transforms-to-execute"Given a workspace, graph, and an optional filter, return the global and workspace definitions to run, in order."[{ws-id :id :as workspace} graph & {:keys [stale-only?]}]- (let [graph (cond->> graph- stale-only? (with-staleness workspace))- entities (:entities graph)- entities (if stale-only?- (filter :stale entities)- entities)- type->ids (u/group-by :node-type :id entities)- id->tx (merge- {}- (when-let [ids (seq (type->ids :external-transform))]- (t2/select-fn->fn :id identity- [:model/Transform :id :name :source :target]- :id [:in ids]))- (when-let [ref-ids (seq (type->ids :workspace-transform))]- (t2/select-fn->fn :ref_id identity- [:model/WorkspaceTransform :ref_id :name :source :target]- :workspace_id ws-id- :ref_id [:in ref-ids])))]- (keep (comp id->tx :id) entities)))+ (let [graph (cond->> graph+ stale-only? (with-staleness workspace))+ entities (cond->> (:entities graph)+ stale-only? (filter :stale))]+ (entities->transforms ws-id entities)))(defn- id->str [ref-id-or-id](if (string? ref-id-or-id)@@ -781,3 +816,59 @@:failed []:not_run []}(transforms-to-execute workspace graph {:stale-only? stale-only?}))))++(defn- stale-ancestors-to-execute+ "Given a workspace, graph, and target ref-id, return the stale ancestor transforms to run, in dependency order.+ Only returns transforms that are both ancestors of the target AND stale.+ Includes both workspace transforms and external transforms in the ancestor chain."+ [{ws-id :id} graph ref-id]+ (let [ancestor-keys (upstream-nodes graph ref-id)+ ancestor? (fn [entity] (contains? ancestor-keys (select-keys entity [:node-type :id])))+ ancestor-entities (filter ancestor? (:entities graph))+ ;; Build a subgraph with only ancestor entities and their inter-dependencies+ ;; Dependencies map is keyed by entities (maps with :id), not by IDs directly+ ancestor-deps (into {}+ (keep (fn [[to-entity from-list]]+ (when (ancestor? to-entity)+ [to-entity (filterv ancestor? from-list)]))+ (:dependencies graph)))+ ancestor-graph {:entities ancestor-entities+ :dependencies ancestor-deps}+ ;; Only fetch staleness for ancestors, not the entire graph+ ancestor-ref-ids (workspace-transform-ids ancestor-entities)+ staleness-map (fetch-staleness-map ws-id ancestor-ref-ids)+ annotated-graph (annotate-staleness ancestor-graph staleness-map)+ stale-entities (filter :stale (:entities annotated-graph))]+ (entities->transforms ws-id stale-entities)))++(defn run-stale-ancestors!+ "Run all stale ancestors of a given transform in dependency order.+ Stops on first failure, marking remaining transforms as not_run.+ Returns a map with :succeeded, :failed, and :not_run lists of ref-ids/transform-ids."+ [workspace graph ref-id]+ (let [ws-id (:id workspace)+ remapping (build-remapping workspace graph)]+ (reduce+ (fn [acc {external-id :id tx-ref-id :ref_id :as transform}]+ (let [node-type (if external-id :external-transform :workspace-transform)+ id-str (id->str (or external-id tx-ref-id))]+ (if (seq (:failed acc))+ ;; Skip execution if any previous transform failed+ (update acc :not_run conj id-str)+ (let [result (try+ (run-transform! workspace graph transform remapping)+ (catch Exception e+ {:status :failed+ :message (ex-message e)+ :error e}))]+ (if (= :succeeded (:status result))+ (update acc :succeeded conj id-str)+ (let [log-data {:workspace-id ws-id :node-type node-type :id id-str :result (dissoc result :error)}]+ (if-let [e (:error result)]+ (log/error e "Failed to execute ancestor transform" log-data)+ (log/error "Failed to execute ancestor transform" log-data))+ (update acc :failed conj id-str)))))))+ {:succeeded []+ :failed []+ :not_run []}+ (stale-ancestors-to-execute workspace graph ref-id))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj b/enterprise/backend/src/metabase_enterprise/workspaces/types.cljindex a8408ce2c49..6ed2e5e69cf 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj@@ -7,6 +7,10 @@(mr/def ::ref-id [:string {:min 1 :api/regex #".+"}])+(mr/def ::flag+ "Boolean flag that accepts true, false, or 1 (for query param compatibility)."+ [:or [:= 1] :boolean])+(def entity-types"The kinds of entities we can store within a Workspace."[{:name "Transform"@@ -22,6 +26,13 @@(mr/def ::entity-map[:map-of ::entity-grouping [:sequential ms/PositiveInt]])+(mr/def ::ancestors-result+ "Result of running stale ancestor transforms."+ [:map+ [:succeeded [:sequential ::ref-id]]+ [:failed [:sequential ::ref-id]]+ [:not_run [:sequential ::ref-id]]])+(mr/def ::execution-result[:map[:status [:enum :succeeded :failed]]@@ -30,7 +41,8 @@[:message {:optional true} [:maybe :string]][:table [:map[:name :string]- [:schema {:optional true} [:maybe :string]]]]])+ [:schema {:optional true} [:maybe :string]]]]+ [:ancestors {:optional true} ::ancestors-result]])(mr/def ::dry-run-result"Result of a transform dry-run (preview without persisting).@@ -43,7 +55,8 @@[:rows {:optional true} [:sequential :any]][:cols {:optional true} [:sequential :map]][:results_metadata {:optional true} [:map- [:columns {:optional true} [:sequential :map]]]]]]])+ [:columns {:optional true} [:sequential :map]]]]]]+ [:ancestors {:optional true} ::ancestors-result]]);;; ---------------------------------------- Graph/Problem Types ----------------------------------------diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex 2d5c792ae65..215a22349a8 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -5,6 +5,7 @@[java-time.api :as t][medley.core :as m][metabase-enterprise.workspaces.common :as ws.common]+ [metabase-enterprise.workspaces.execute :as ws.execute][metabase-enterprise.workspaces.impl :as ws.impl][metabase-enterprise.workspaces.isolation :as ws.isolation][metabase-enterprise.workspaces.test-util :as ws.tu]@@ -23,7 +24,9 @@[metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]][metabase.util :as u]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.time Instant)))(set! *warn-on-reflection* true)@@ -1299,6 +1302,74 @@:message string?}(mt/user-http-request :crowberto :post 200 (ws-url (:id ws1) "transform" ref-id "dry-run"))))))))))+(deftest run-transform-with-stale-ancestors-test+ (testing "POST /api/ee/workspace/:id/transform/:txid/run with run_stale_ancestors=true"+ ;; Chain: x1 -> x2 -> x3, where x1 is stale+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}+ :properties {:x1 {:definition_changed true}+ :x2 {:definition_changed false}+ :x3 {:definition_changed false}}}}]+ (ws.tu/ws-done! workspace-id)+ (let [x1-ref (workspace-map :x1)+ x2-ref (workspace-map :x2)+ x3-ref (workspace-map :x3)]+ ;; Use a mock that returns :table (required by the API schema)+ (mt/with-dynamic-fn-redefs [ws.execute/run-transform-with-remapping+ (fn [{:keys [target]} _remapping]+ {:status :succeeded+ :end_time (Instant/now)+ :message "Mocked execution"+ :table (select-keys target [:schema :name])})]+ (testing "without flag, ancestors are not run"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url workspace-id "/transform/" x3-ref "/run"))]+ (is (= "succeeded" (:status result)))+ (is (nil? (:ancestors result)))))+ (testing "with flag, stale ancestors are run before target"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url workspace-id "/transform/" x3-ref "/run")+ {:run_stale_ancestors true})]+ (is (= "succeeded" (:status result)))+ (is (= {:succeeded [x1-ref x2-ref]+ :failed []+ :not_run []}+ (:ancestors result))))))))))++(deftest dry-run-transform-with-stale-ancestors-test+ (testing "POST /api/ee/workspace/:id/transform/:txid/dry-run with run_stale_ancestors=true"+ ;; Chain: x1 -> x2 -> x3, where x1 is stale+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}+ :properties {:x1 {:definition_changed true}+ :x2 {:definition_changed false}+ :x3 {:definition_changed false}}}}]+ (ws.tu/ws-done! workspace-id)+ (let [x1-ref (workspace-map :x1)+ x2-ref (workspace-map :x2)+ x3-ref (workspace-map :x3)]+ ;; Mock both run (for ancestors) and preview (for dry-run target)+ (mt/with-dynamic-fn-redefs [ws.execute/run-transform-with-remapping+ (fn [{:keys [target]} _remapping]+ {:status :succeeded+ :end_time (Instant/now)+ :message "Mocked execution"+ :table (select-keys target [:schema :name])})+ ws.execute/run-transform-preview+ (fn [_transform _remapping]+ {:status :succeeded+ :data {:rows [[1 "test"]]+ :cols [{:name "id"} {:name "name"}]}})]+ (testing "with flag, stale ancestors are run before dry-run"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url workspace-id "/transform/" x3-ref "/dry-run")+ {:run_stale_ancestors true})]+ (is (= "succeeded" (:status result)))+ (is (= {:succeeded [x1-ref x2-ref]+ :failed []+ :not_run []}+ (:ancestors result))))))))))+(defn- random-target [db-id]{:type "table":database db-iddiff --git a/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.cljindex e78ca914381..c35891396b8 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/impl_test.clj@@ -358,9 +358,9 @@(is (true? (:definition_changed entity)))(is (false? (:input_data_changed entity))))))-;;;; Static graph tests for marking functions (upstream-ancestors, downstream-descendants, any-ancestor-stale?)+;;;; Static graph tests for graph traversal functions-(deftest upstream-ancestors-linear-chain-test+(deftest upstream-ids-linear-chain-test(testing "Linear chain: computes all upstream ancestors"(let [t1 (workspace-transform "t1")t2 (workspace-transform "t2")@@ -371,14 +371,14 @@t3 [t2]}}](testing "t3 has t1 and t2 as ancestors"(is (= #{"t1" "t2"}- (set (ws.impl/upstream-ancestors graph "t3")))))+ (set (ws.impl/upstream-ids graph :workspace-transform "t3")))))(testing "t2 has only t1 as ancestor"(is (= #{"t1"}- (set (ws.impl/upstream-ancestors graph "t2")))))+ (set (ws.impl/upstream-ids graph :workspace-transform "t2")))))(testing "t1 has no ancestors"- (is (empty? (ws.impl/upstream-ancestors graph "t1")))))))+ (is (empty? (ws.impl/upstream-ids graph :workspace-transform "t1")))))))-(deftest upstream-ancestors-diamond-test+(deftest upstream-ids-diamond-test(testing "Diamond graph: computes all paths"(let [t1 (workspace-transform "t1")t2 (workspace-transform "t2")@@ -395,13 +395,13 @@t4 [t2 t3]}}](testing "t4 has all transforms as ancestors"(is (= #{"t1" "t2" "t3"}- (set (ws.impl/upstream-ancestors graph "t4")))))+ (set (ws.impl/upstream-ids graph :workspace-transform "t4")))))(testing "t2 has only t1 as ancestor"(is (= #{"t1"}- (set (ws.impl/upstream-ancestors graph "t2"))))))))+ (set (ws.impl/upstream-ids graph :workspace-transform "t2"))))))))-(deftest upstream-ancestors-with-external-transforms-test- (testing "External transforms are traversed but not included"+(deftest upstream-ids-with-external-transforms-test+ (testing "External transforms are traversed but not included when filtered"(let [t1 (workspace-transform "t1")ext (global-transform 100)t2 (workspace-transform "t2")@@ -411,11 +411,13 @@t2 [ext]}}](testing "t2 can reach through external transform to t1"(is (= #{"t1"}- (set (ws.impl/upstream-ancestors graph "t2")))))- (testing "ext cannot be returned as it's external"- (is (not (contains? (set (ws.impl/upstream-ancestors graph "t2")) 100)))))))+ (set (ws.impl/upstream-ids graph :workspace-transform "t2")))))+ (testing "upstream-nodes returns all node types including external"+ (is (= #{{:node-type :workspace-transform :id "t1"}+ {:node-type :external-transform :id 100}}+ (ws.impl/upstream-nodes graph "t2")))))))-(deftest downstream-descendants-linear-chain-test+(deftest downstream-ids-linear-chain-test(testing "Linear chain: computes all downstream descendants"(let [t1 (workspace-transform "t1")t2 (workspace-transform "t2")@@ -426,14 +428,14 @@t3 [t2]}}](testing "t1 has t2 and t3 as descendants"(is (= #{"t2" "t3"}- (set (ws.impl/downstream-descendants graph "t1")))))+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))(testing "t2 has only t3 as descendant"(is (= #{"t3"}- (set (ws.impl/downstream-descendants graph "t2")))))+ (set (ws.impl/downstream-ids graph :workspace-transform "t2")))))(testing "t3 has no descendants"- (is (empty? (ws.impl/downstream-descendants graph "t3")))))))+ (is (empty? (ws.impl/downstream-ids graph :workspace-transform "t3")))))))-(deftest downstream-descendants-diamond-test+(deftest downstream-ids-diamond-test(testing "Diamond graph: computes all downstream paths"(let [t1 (workspace-transform "t1")t2 (workspace-transform "t2")@@ -450,13 +452,13 @@t4 [t2 t3]}}](testing "t1 has all transforms as descendants"(is (= #{"t2" "t3" "t4"}- (set (ws.impl/downstream-descendants graph "t1")))))+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))(testing "t2 has only t4 as descendant"(is (= #{"t4"}- (set (ws.impl/downstream-descendants graph "t2"))))))))+ (set (ws.impl/downstream-ids graph :workspace-transform "t2"))))))))-(deftest downstream-descendants-with-external-transforms-test- (testing "External transforms are traversed but not included"+(deftest downstream-ids-with-external-transforms-test+ (testing "External transforms are traversed but not included when filtered"(let [t1 (workspace-transform "t1")ext (global-transform 100)t2 (workspace-transform "t2")@@ -466,14 +468,143 @@t2 [ext]}}](testing "t1 can reach through external transform to t2"(is (= #{"t2"}- (set (ws.impl/downstream-descendants graph "t1")))))- (testing "ext cannot be returned as it's external"- (is (not (contains? (set (ws.impl/downstream-descendants graph "t1")) 100)))))))+ (set (ws.impl/downstream-ids graph :workspace-transform "t1")))))+ (testing "downstream-nodes returns all node types including external"+ (is (= #{{:node-type :workspace-transform, :id "t2"}+ {:node-type :external-transform, :id 100}}+ (ws.impl/downstream-nodes graph "t1")))))))-(deftest downstream-descendants-empty-graph-test+(deftest downstream-nodes-empty-graph-test(testing "Empty graph returns nil"- (is (nil? (ws.impl/downstream-descendants {:entities [] :dependencies nil} "t1")))))+ (is (nil? (ws.impl/downstream-nodes {:entities [] :dependencies nil} "t1")))))-(deftest upstream-ancestors-empty-graph-test+(deftest upstream-nodes-empty-graph-test(testing "Empty graph returns nil"- (is (nil? (ws.impl/upstream-ancestors {:entities [] :dependencies nil} "t1")))))+ (is (nil? (ws.impl/upstream-nodes {:entities [] :dependencies nil} "t1")))))++;;;; run-stale-ancestors! tests++(deftest run-stale-ancestors-runs-only-stale-ancestors-test+ (testing "run-stale-ancestors! only runs ancestors that are stale (including transitively stale)"+ ;; Graph:+ ;; x1 -> x2 \+ ;; x3 -> x4 -> x7+ ;; x5 -> x6 /+ ;; x1 and x4 are locally stale, x2 is transitively stale (depends on x1)+ ;; x3, x5, x6 are fresh and should NOT run+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1]+ :x3 [:t0] :x4 [:x3]+ :x5 [:t0] :x6 [:x5]+ :x7 [:x2 :x4 :x6]}+ :properties {:x1 {:definition_changed true}+ :x2 {:definition_changed false}+ :x3 {:definition_changed false}+ :x4 {:definition_changed true}+ :x5 {:definition_changed false}+ :x6 {:definition_changed false}+ :x7 {:definition_changed false}}}}]+ (let [t1-ref (workspace-map :x1)+ t2-ref (workspace-map :x2)+ t4-ref (workspace-map :x4)+ t7-ref (workspace-map :x7)]+ (ws.tu/with-mocked-execution+ (let [workspace (t2/select-one :model/Workspace workspace-id)+ graph (ws.impl/get-or-calculate-graph! workspace)+ result (ws.impl/run-stale-ancestors! workspace graph t7-ref)+ succeeded (:succeeded result)]+ (testing "x1, x2 (transitively stale), and x4 run; x3, x5, x6 do not"+ (is (= (set [t1-ref t2-ref t4-ref]) (set succeeded)))+ (is (= [] (:failed result)))+ (is (= [] (:not_run result))))+ (testing "x1 runs before x2 (dependency order within branch)"+ (is (< (.indexOf succeeded t1-ref) (.indexOf succeeded t2-ref))))))))))++(deftest run-stale-ancestors-respects-dependency-order-test+ (testing "run-stale-ancestors! runs ancestors in dependency order"+ ;; Chain: t1 -> t2 -> t3, where t1 and t2 are stale+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}+ :properties {:x1 {:definition_changed true}+ :x2 {:definition_changed true}+ :x3 {:definition_changed false}}}}]+ (let [t1-ref (workspace-map :x1)+ t2-ref (workspace-map :x2)+ t3-ref (workspace-map :x3)+ run-order (atom [])]+ (mt/with-dynamic-fn-redefs [ws.impl/run-transform!+ (fn [_ws _graph transform & _]+ (swap! run-order conj (:ref_id transform))+ {:status :succeeded})]+ (let [workspace (t2/select-one :model/Workspace workspace-id)+ graph (ws.impl/get-or-calculate-graph! workspace)+ result (ws.impl/run-stale-ancestors! workspace graph t3-ref)]+ (testing "both stale ancestors are run in order (t1 before t2)"+ (is (= [t1-ref t2-ref] @run-order))+ (is (= [t1-ref t2-ref] (:succeeded result))))))))))++(deftest run-stale-ancestors-stops-on-failure-test+ (testing "run-stale-ancestors! stops on first failure and marks remaining as not_run"+ ;; Chain: t1 -> t2 -> t3, where t1 and t2 are stale, t1 fails+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1] :x3 [:x2]}+ :properties {:x1 {:definition_changed true}+ :x2 {:definition_changed true}+ :x3 {:definition_changed false}}}}]+ (let [t1-ref (workspace-map :x1)+ t2-ref (workspace-map :x2)+ t3-ref (workspace-map :x3)]+ (mt/with-dynamic-fn-redefs [ws.impl/run-transform!+ (fn [_ws _graph transform & _]+ (if (= (:ref_id transform) t1-ref)+ {:status :failed :message "Simulated failure"}+ {:status :succeeded}))]+ (let [workspace (t2/select-one :model/Workspace workspace-id)+ graph (ws.impl/get-or-calculate-graph! workspace)+ result (ws.impl/run-stale-ancestors! workspace graph t3-ref)]+ (testing "t1 fails, t2 is not_run"+ (is (= [] (:succeeded result)))+ (is (= [t1-ref] (:failed result)))+ (is (= [t2-ref] (:not_run result))))))))))++(deftest run-stale-ancestors-no-stale-ancestors-test+ (testing "run-stale-ancestors! returns empty results when no ancestors are stale"+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 [:t0] :x2 [:x1]}+ :properties {:x1 {:definition_changed false}+ :x2 {:definition_changed false}}}}]+ (let [t2-ref (workspace-map :x2)]+ (ws.tu/with-mocked-execution+ (let [workspace (t2/select-one :model/Workspace workspace-id)+ graph (ws.impl/get-or-calculate-graph! workspace)+ result (ws.impl/run-stale-ancestors! workspace graph t2-ref)]+ (testing "no transforms are run"+ (is (= [] (:succeeded result)))+ (is (= [] (:failed result)))+ (is (= [] (:not_run result))))))))))++(deftest run-stale-ancestors-includes-external-transforms-test+ (testing "run-stale-ancestors! includes enclosed external transforms in the ancestor chain"+ ;; Graph: (x1) -> x2 -> (x3) where x2 is an external (global) transform+ ;; x1 is a workspace checkout, x2 is enclosed (global, not checked out), x3 is a workspace checkout+ ;; x1 is locally stale -> x2 becomes transitively stale -> running ancestors for x3 should run both x1 and x2+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map global-map]}+ {:global {:x1 [:t0] :x2 [:x1] :x3 [:x2]}+ :workspace {:checkouts [:x1 :x3]+ :properties {:x1 {:definition_changed true}+ :x3 {:definition_changed false}}}}]+ (ws.tu/ws-done! workspace-id)+ (let [x1-ref (workspace-map :x1)+ x2-global (str "global-id:" (global-map :x2))+ x3-ref (workspace-map :x3)]+ (ws.tu/with-mocked-execution+ (let [workspace (t2/select-one :model/Workspace workspace-id)+ graph (ws.impl/get-or-calculate-graph! workspace)+ result (ws.impl/run-stale-ancestors! workspace graph x3-ref)+ succeeded (:succeeded result)]+ (testing "both workspace transform x1 and external transform x2 are run"+ (is (= #{x1-ref x2-global} (set succeeded)))+ (is (= [] (:failed result)))+ (is (= [] (:not_run result))))+ (testing "x1 runs before x2 (dependency order)"+ (is (< (.indexOf succeeded x1-ref) (.indexOf succeeded x2-global))))))))))