The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Transform run dag be (#76974)
Nicola Mometto30021f1abd12
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 216b43e77e2..bc4ebdbcb70 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -526,6 +526,7 @@metabase.test/with-temp-empty-app-db clojure.core/letmetabase.test/with-temp-file clojure.core/letmetabase.test/with-user-in-groups clojure.core/let+ metabase.transforms-rest.api.transform-runs-test/with-runs-fixture! clojure.core/fnmetabase.upload.impl-test/with-upload-table! clojure.core/letmetabase.util/for-map clojure.core/formetabase.util.files/with-open-path-to-resource clojure.core/letdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 02f0f0d4340..19d4a54f271 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2447,6 +2447,7 @@:friends #{modelsenterprise/transforms-python}:model-exports #{:model/Transform+ :model/TransformDagRun:model/TransformJob:model/TransformJobRun:model/TransformJobTransformTag@@ -2504,6 +2505,7 @@transforms-baseutil}:model-imports #{:model/Transform+ :model/TransformDagRun:model/TransformJob:model/TransformJobRun:model/TransformJobTransformTagdiff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex 62c9b93f793..7629b344b3d 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -1919,7 +1919,7 @@ LIMITgetRunStatus().should("have.text", "Run in progress…");getCancelButton().click();- H.modal().button("Yes").click();+ H.modal().button("Cancel run").click();getRunButton().should("have.text", "Canceling…");getRunStatus().should("have.text", "Canceling…");@@ -1938,6 +1938,7 @@ LIMITgetRunStatus().should("have.text", "Run in progress…");getRunsNavLink().click();+ getDetailedViewSwitch().click();getTransformRunTable().findByText("In progress").click();cy.findByTestId("run-list-sidebar").button("Cancel run").click();H.modal().button("Yes").click();@@ -1956,7 +1957,7 @@ LIMITgetRunStatus().should("have.text", "Run in progress…");getCancelButton().click();- H.modal().button("Yes").click();+ H.modal().button("Cancel run").click();getRunButton().should("have.text", "Canceling…");getRunStatus().should("have.text", "Canceling…");@@ -3110,6 +3111,7 @@ describe("scenarios > admin > transforms > jobs", () => {.should("be.visible");getRunsNavLink().click();+ getDetailedViewSwitch().click();getTransformRunTable().within(() => {cy.findByText("MBQL transform").should("be.visible");cy.findByText("Success").should("be.visible");@@ -3651,6 +3653,7 @@ describe("scenarios > admin > transforms > runs", () => {createInitialData();getRunsNavLink().click();+ getDetailedViewSwitch().click();testTransformFilter();testStatusFilter();testTagFilter();@@ -3700,6 +3703,7 @@ describe("scenarios > admin > transforms > runs", () => {createInitialData();getRunsNavLink().click();+ getDetailedViewSwitch().click();// ascending: "MBQL transform" < "SQL transform"testSorting({@@ -4033,7 +4037,11 @@ function openBulkActionsMenu() {}function visitRunListPage() {- return cy.visit("/data-studio/transforms/runs");+ return cy.visit("/data-studio/transforms/runs/individual");+}++function getDetailedViewSwitch() {+ return cy.findByTestId("detailed-view-switch");}function runTransformAndWaitForSuccess() {diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex b941d926289..7dfe45dbdc4 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -153,6 +153,7 @@"TaskHistory""TaskRun""Tenant"+ "TransformDagRun""TransformJobRun""TransformRun""TransformRunCancelation"diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 8315836fff5..e2ace811569 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -97,7 +97,7 @@"Execute a Python transform by calling the python runner.Blocks until the transform returns."- [transform {:keys [run-method on-start user-id job-run-id]}]+ [transform {:keys [run-method on-start user-id parent-run]}](try(let [message-log (base/empty-message-log){:keys [target owner_user_id creator_id] transform-id :id} transform@@ -105,7 +105,8 @@run-user-id (if (and (= run-method :manual) user-id)user-id(or owner_user_id creator_id))- {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id :job-run-id job-run-id)]+ {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id+ :parent-run parent-run)](when on-start (on-start run-id))(with-open [_ (open-python-message-update-future! run-id message-log)](driver.conn/with-write-connectiondiff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 38bb58f3360..b70654cffc9 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -134,6 +134,7 @@;; TODO we should remove these models from here once serialization is supported:model/TransformRun:model/TransformRunCancelation+ :model/TransformDagRun:model/TransformJobRun:model/TransformJobTransformTag:model/TransformTransformTagdiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.cljindex 0e42e850901..1f013bc5733 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj@@ -62,7 +62,12 @@(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message](swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)]- (#'jobs/run-transform! run-id :scheduled nil (promise) python-transform)+ (#'jobs/run-transform! {:parent-run [:job run-id]+ :run-method :scheduled+ :user-id nil+ :add-run-activity! (constantly nil)}+ (promise)+ python-transform)(is (= 1 (count @logged-messages))"Should log exactly one warning")(is (= :warn (:level (first @logged-messages)))diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex 47e34fd89c7..c02f5eb3ef9 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -2,10 +2,12 @@ import type {DatabaseId,InspectorCard,InspectorSource,+ ListTransformGraphRunsResponse,ListTransformJobRunsResponse,ListTransformRunsResponse,PythonTransformTableAliases,Transform,+ TransformGraphRun,TransformJob,TransformJobRun,TransformOwner,@@ -185,6 +187,39 @@ export function createMockTransformRunForJobRun(};}+export function createMockTransformGraphRun(+ opts?: Partial<TransformGraphRun>,+): TransformGraphRun {+ return {+ run_type: "job",+ id: 1,+ entity_id: 1,+ name: "Graph run",+ direction: null,+ transform_count: null,+ run_method: "manual",+ status: "succeeded",+ is_active: false,+ start_time: "2024-01-01T00:00:00Z",+ end_time: "2024-01-01T00:01:00Z",+ message: null,+ user_id: null,+ ...opts,+ };+}++export function createMockListTransformGraphRunsResponse(+ opts?: Partial<ListTransformGraphRunsResponse>,+): ListTransformGraphRunsResponse {+ return {+ data: [],+ total: 0,+ limit: null,+ offset: null,+ ...opts,+ };+}+export function createMockTransformTag(opts?: Partial<TransformTag>,): TransformTag {diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 99c055c9a02..6fb5dfab6dd 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -379,6 +379,79 @@ export type ListJobRunTransformRunsRequest = {runId: TransformJobRunId;};+export type CancelJobRunRequest = {+ jobId: TransformJobId;+ runId: TransformJobRunId;+};++export type TransformDagRunId = number;++export const TRANSFORM_DAG_DIRECTIONS = ["upstream", "downstream"] as const;+export type TransformDagDirection = (typeof TRANSFORM_DAG_DIRECTIONS)[number];++export type RunTransformDagRequest = {+ id: TransformId;+ direction: TransformDagDirection;+ skip_fresh_deps?: boolean;+};++export type RunTransformDagResponse = {+ dag_run_id: TransformDagRunId | null;+ message: string;+};++export type ListDagTransformsRequest = {+ transformId: TransformId;+ direction: TransformDagDirection;+};++export type DagTransform = Pick<Transform, "id" | "name">;++export type ListDagRunTransformRunsRequest = {+ dagRunId: TransformDagRunId;+};++export const TRANSFORM_GRAPH_RUN_TYPES = ["job", "dag", "transform"] as const;+export type TransformGraphRunType = (typeof TRANSFORM_GRAPH_RUN_TYPES)[number];++export const TRANSFORM_GRAPH_RUN_SORT_COLUMNS = [+ "start_time",+ "end_time",+] as const;+export type TransformGraphRunSortColumn =+ (typeof TRANSFORM_GRAPH_RUN_SORT_COLUMNS)[number];++export type TransformGraphRun = {+ run_type: TransformGraphRunType;+ id: number;+ entity_id: number | null;+ name: string | null;+ direction: TransformDagDirection | null;+ transform_count: number | null;+ run_method: TransformRunMethod | null;+ status: TransformRunStatus | null;+ is_active: boolean | null;+ start_time: string;+ end_time: string | null;+ message: string | null;+ user_id: UserId | null;+};++export type ListTransformGraphRunsRequest = {+ types?: TransformGraphRunType[];+ statuses?: TransformRunStatus[];+ "transform-ids"?: TransformId[];+ "start-time"?: string;+ "end-time"?: string;+ "run-methods"?: TransformRunMethod[];+ "sort-column"?: TransformGraphRunSortColumn;+ "sort-direction"?: SortDirection;+} & PaginationRequest;++export type ListTransformGraphRunsResponse = {+ data: TransformGraphRun[];+} & PaginationResponse;+export type TestPythonTransformRequest = {code: string;source_tables: PythonTransformTableAliases;diff --git a/frontend/src/metabase/api/tags/constants.ts b/frontend/src/metabase/api/tags/constants.tsindex 25b9167f7d1..a5ba21c986b 100644--- a/frontend/src/metabase/api/tags/constants.ts+++ b/frontend/src/metabase/api/tags/constants.ts@@ -61,6 +61,7 @@ export const TAG_TYPES = ["transform-job","transform-job-via-tag","transform-run",+ "transform-dag-run","transform-inspector-lens","user","public-dashboard",diff --git a/frontend/src/metabase/api/transform-job.ts b/frontend/src/metabase/api/transform-job.tsindex 1164e5c01f6..67cf0828e43 100644--- a/frontend/src/metabase/api/transform-job.ts+++ b/frontend/src/metabase/api/transform-job.ts@@ -82,6 +82,8 @@ export const transformJobApi = Api.injectEndpoints({invalidatesTags: (_, error, id) =>invalidateTags(error, [idTag("transform-job", id),+ listTag("transform-job"),+ listTag("transform-run"),tag("transform"),tag("table"),listTag("table-remapping"),diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.tsindex 266c21b491c..3690054d8d1 100644--- a/frontend/src/metabase/api/transform.ts+++ b/frontend/src/metabase/api/transform.ts@@ -1,17 +1,27 @@import { isResourceNotFoundError } from "metabase/utils/errors";import type {+ CancelJobRunRequest,CreateTransformRequest,+ DagTransform,Dataset,GetInspectorLensRequest,InspectorDiscoveryResponse,InspectorLens,+ ListDagRunTransformRunsRequest,+ ListDagTransformsRequest,+ ListTransformGraphRunsRequest,+ ListTransformGraphRunsResponse,ListTransformRunsRequest,ListTransformRunsResponse,ListTransformsRequest,RunInspectorQueryRequest,+ RunTransformDagRequest,+ RunTransformDagResponse,RunTransformResponse,Transform,+ TransformDagRunId,TransformId,+ TransformRunForJobRun,UpdateTransformRequest,} from "metabase-types/api";@@ -106,7 +116,11 @@ export const transformApi = Api.injectEndpoints({url: `/api/transform/${id}/cancel`,}),invalidatesTags: (_, error, id) =>- invalidateTags(error, [idTag("transform", id), tag("table")]),+ invalidateTags(error, [+ idTag("transform", id),+ tag("table"),+ listTag("transform-run"),+ ]),onQueryStarted: async (id, { dispatch, queryFulfilled }) => {const patchResult = dispatch(transformApi.util.updateQueryData("getTransform", id, (draft) => {@@ -139,6 +153,88 @@ export const transformApi = Api.injectEndpoints({}},}),+ runTransformDag: builder.mutation<+ RunTransformDagResponse,+ RunTransformDagRequest+ >({+ query: ({ id, ...body }) => ({+ method: "POST",+ url: `/api/transform/${id}/run-dag`,+ body,+ }),+ invalidatesTags: (_, error, { id }) =>+ invalidateTags(error, [+ idTag("transform", id),+ tag("transform"),+ tag("table"),+ listTag("table-remapping"),+ listTag("transform-run"),+ listTag("transform-dag-run"),+ ]),+ }),+ listDagTransforms: builder.query<DagTransform[], ListDagTransformsRequest>({+ query: ({ transformId, direction }) => ({+ method: "GET",+ url: `/api/transform/${transformId}/dag-transforms`,+ params: { direction },+ }),+ providesTags: (_response, _error, { transformId }) => [+ idTag("transform", transformId),+ ],+ }),+ listDagRunTransformRuns: builder.query<+ TransformRunForJobRun[],+ ListDagRunTransformRunsRequest+ >({+ query: ({ dagRunId }) => ({+ method: "GET",+ url: `/api/transform-dag-run/${dagRunId}/transform-runs`,+ }),+ providesTags: (_response, _error, { dagRunId }) => [+ idTag("transform-dag-run", dagRunId),+ ],+ }),+ cancelDagRun: builder.mutation<void, TransformDagRunId>({+ query: (runId) => ({+ method: "POST",+ url: `/api/transform-dag-run/${runId}/cancel`,+ }),+ invalidatesTags: (_, error, runId) =>+ invalidateTags(error, [+ idTag("transform-dag-run", runId),+ listTag("transform-dag-run"),+ listTag("transform-run"),+ tag("transform"),+ ]),+ }),+ cancelJobRun: builder.mutation<void, CancelJobRunRequest>({+ query: ({ jobId, runId }) => ({+ method: "POST",+ url: `/api/transform-job/${jobId}/runs/${runId}/cancel`,+ }),+ invalidatesTags: (_, error, { jobId }) =>+ invalidateTags(error, [+ idTag("transform-job", jobId),+ listTag("transform-job"),+ listTag("transform-run"),+ tag("transform"),+ ]),+ }),+ listTransformGraphRuns: builder.query<+ ListTransformGraphRunsResponse,+ ListTransformGraphRunsRequest+ >({+ query: (params) => ({+ method: "GET",+ url: "/api/transform/runs",+ params,+ }),+ providesTags: [+ listTag("transform-run"),+ listTag("transform-dag-run"),+ listTag("transform-job"),+ ],+ }),createTransform: builder.mutation<Transform, CreateTransformRequest>({query: (body) => ({method: "POST",@@ -190,7 +286,12 @@ export const transformApi = Api.injectEndpoints({url: `/api/transform/${id}`,}),invalidatesTags: (_, error, id) =>- invalidateTags(error, [listTag("transform"), idTag("transform", id)]),+ invalidateTags(error, [+ listTag("transform"),+ idTag("transform", id),+ listTag("transform-run"),+ listTag("transform-dag-run"),+ ]),}),deleteTransformTarget: builder.mutation<void, TransformId>({query: (id) => ({@@ -245,6 +346,12 @@ export const {useGetInspectorLensQuery,useLazyGetInspectorLensQuery,useRunTransformMutation,+ useRunTransformDagMutation,+ useListDagTransformsQuery,+ useListDagRunTransformRunsQuery,+ useCancelDagRunMutation,+ useCancelJobRunMutation,+ useListTransformGraphRunsQuery,useCancelCurrentTransformRunMutation,useCreateTransformMutation,useUpdateTransformMutation,diff --git a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsxindex e6af54d6d90..9d97c0a953d 100644--- a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx+++ b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx@@ -233,7 +233,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {<DataStudioTablabel={t`Runs`}icon="play_outlined"- to={Urls.transformRunList()}+ to={Urls.transformGraphRunList()}isSelected={currentTab === "runs"}showLabel={isNavbarOpened}/>diff --git a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.tsindex d0cb7ce5c1d..ee52532f714 100644--- a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts+++ b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts@@ -32,7 +32,7 @@ export const getCurrentTab = (pathname: string): TabName => {return "schema-viewer";case pathname.startsWith(Urls.dataStudioLibrary()):return "library";- case pathname.startsWith(Urls.transformRunList()):+ case pathname.startsWith(Urls.transformGraphRunList()):return "runs";case pathname.startsWith(Urls.transformList()):return "transforms";diff --git a/frontend/src/metabase/transforms/analytics.ts b/frontend/src/metabase/transforms/analytics.tsindex d7be9613167..e7aa306069f 100644--- a/frontend/src/metabase/transforms/analytics.ts+++ b/frontend/src/metabase/transforms/analytics.ts@@ -3,6 +3,7 @@ import type {IndexKind,InspectorCardId,InspectorLensId,+ TransformDagDirection,TransformId,TransformJobId,} from "metabase-types/api";@@ -21,6 +22,20 @@ export function trackTransformTriggerManualRun({});}+export function trackTransformTriggerDagRun({+ transformId,+ direction,+}: {+ transformId: TransformId;+ direction: TransformDagDirection;+}) {+ trackSimpleEvent({+ event: "transform_trigger_dag_run",+ target_id: transformId,+ event_detail: direction,+ });+}+export function trackTransformJobTriggerManualRun({jobId,}: {@@ -32,6 +47,17 @@ export function trackTransformJobTriggerManualRun({});}+export function trackTransformRunsViewToggled({+ view,+}: {+ view: "detailed" | "grouped";+}) {+ trackSimpleEvent({+ event: "transform_runs_view_toggled",+ event_detail: view,+ });+}+export function trackTransformCreate({creationType,}: {diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.css b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.cssnew file mode 100644index 00000000000..5b1788f5547--- /dev/null+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.css@@ -0,0 +1,3 @@+.label {+ white-space: nowrap;+}diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsx b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsxnew file mode 100644index 00000000000..1e5bfee3672--- /dev/null+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsx@@ -0,0 +1,44 @@+import { t } from "ttag";++import { useDispatch } from "metabase/redux";+import { push } from "metabase/router";+import { trackTransformRunsViewToggled } from "metabase/transforms/analytics";+import { Switch } from "metabase/ui";+import * as Urls from "metabase/urls";++import S from "./DetailedViewSwitch.module.css";++type DetailedViewSwitchProps = {+ detailed: boolean;+ params: Urls.CommonRunListParams;+};++export function DetailedViewSwitch({+ detailed,+ params,+}: DetailedViewSwitchProps) {+ const dispatch = useDispatch();++ const handleChange = () => {+ const nextView = detailed ? "grouped" : "detailed";+ trackTransformRunsViewToggled({ view: nextView });+ dispatch(+ push(+ detailed+ ? Urls.transformGraphRunList(params)+ : Urls.transformRunList(params),+ ),+ );+ };++ return (+ <Switch+ label={t`Detailed view`}+ labelPosition="left"+ checked={detailed}+ onChange={handleChange}+ classNames={{ label: S.label }}+ data-testid="detailed-view-switch"+ />+ );+}diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsx b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsxnew file mode 100644index 00000000000..1836d05262b--- /dev/null+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsx@@ -0,0 +1,63 @@+import userEvent from "@testing-library/user-event";++import { renderWithProviders, screen } from "__support__/ui";+import { Route } from "metabase/router";+import * as Urls from "metabase/urls";++import { trackTransformRunsViewToggled } from "../../analytics";++import { DetailedViewSwitch } from "./DetailedViewSwitch";++jest.mock("../../analytics", () => ({+ ...jest.requireActual("../../analytics"),+ trackTransformRunsViewToggled: jest.fn(),+}));++type SetupOpts = {+ detailed: boolean;+ params?: Urls.CommonRunListParams;+};++function setup({ detailed, params = {} }: SetupOpts) {+ const initialRoute = detailed+ ? Urls.transformRunList()+ : Urls.transformGraphRunList();++ const { history } = renderWithProviders(+ <Route+ path="*"+ element={<DetailedViewSwitch detailed={detailed} params={params} />}+ />,+ { withRouter: true, initialRoute },+ );++ return { history };+}++describe("DetailedViewSwitch", () => {+ it("tracks a toggle to the detailed view and navigates there", async () => {+ const { history } = setup({ detailed: false });++ await userEvent.click(screen.getByLabelText("Detailed view"));++ expect(trackTransformRunsViewToggled).toHaveBeenCalledWith({+ view: "detailed",+ });+ expect(history?.getCurrentLocation().pathname).toBe(+ "/data-studio/transforms/runs/individual",+ );+ });++ it("tracks a toggle to the grouped view and navigates there", async () => {+ const { history } = setup({ detailed: true });++ await userEvent.click(screen.getByLabelText("Detailed view"));++ expect(trackTransformRunsViewToggled).toHaveBeenCalledWith({+ view: "grouped",+ });+ expect(history?.getCurrentLocation().pathname).toBe(+ "/data-studio/transforms/runs",+ );+ });+});diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.ts b/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.tsnew file mode 100644index 00000000000..ea1b634ed75--- /dev/null+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.ts@@ -0,0 +1 @@+export { DetailedViewSwitch } from "./DetailedViewSwitch";diff --git a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsxindex c3c2262e22c..8234d3b8e19 100644--- a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx+++ b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx@@ -1,12 +1,18 @@+import { useDisclosure } from "@mantine/hooks";import { c, t } from "ttag";-import { useRunTransformJobMutation } from "metabase/api";+import {+ useCancelJobRunMutation,+ useRunTransformJobMutation,+} from "metabase/api";+import { ConfirmModal } from "metabase/common/components/ConfirmModal";import { Schedule } from "metabase/common/components/Schedule";import { TitleSection } from "metabase/common/data-studio/components/TitleSection";import { useSetting } from "metabase/common/hooks";import { useMetadataToasts } from "metabase/metadata/hooks";import { Box, Divider, Group, Tooltip } from "metabase/ui";import { getScheduleExplanation } from "metabase/utils/cron";+import { isResourceNotFoundError } from "metabase/utils/errors";import type {ScheduleDisplayType,ScheduleSettings,@@ -130,7 +136,12 @@ function RunButtonSection({isCheckingPermissions,}: RunButtonSectionProps) {const [runJob] = useRunTransformJobMutation();+ const [cancelJobRun] = useCancelJobRunMutation();const { sendErrorToast } = useMetadataToasts();+ const [+ isCancelModalOpen,+ { open: openCancelModal, close: closeCancelModal },+ ] = useDisclosure(false);const isSaved = job.id != null;const hasTags = job.tag_ids?.length !== 0;@@ -160,15 +171,44 @@ function RunButtonSection({}};+ const handleCancel = async () => {+ if (job.id == null || job.last_run?.id == null) {+ return;+ }+ const { error } = await cancelJobRun({+ jobId: job.id,+ runId: job.last_run.id,+ });+ if (error && !isResourceNotFoundError(error)) {+ sendErrorToast(t`Failed to cancel job`);+ }+ };+return (- <Tooltip label={tooltipLabel} disabled={!tooltipLabel}>- <RunButton- id={job.id}- run={job.last_run}- isDisabled={!isSaved || !hasTags || readOnly}- isLoading={isCheckingPermissions}- onRun={handleRun}+ <>+ <Tooltip label={tooltipLabel} disabled={!tooltipLabel}>+ <RunButton+ id={job.id}+ run={job.last_run}+ isDisabled={!isSaved || !hasTags || readOnly}+ isLoading={isCheckingPermissions}+ allowCancellation+ onRun={handleRun}+ onCancel={openCancelModal}+ />+ </Tooltip>+ <ConfirmModal+ title={t`Cancel this run?`}+ message={t`This stops the job run and requests cancellation of any transforms still in progress. Transforms that have already finished won't be reverted.`}+ confirmButtonText={t`Cancel run`}+ closeButtonText={t`Keep running`}+ opened={isCancelModalOpen}+ onClose={closeCancelModal}+ onConfirm={() => {+ void handleCancel();+ closeCancelModal();+ }}/>- </Tooltip>+ </>);}diff --git a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsxindex 830b40e0cfc..3b5917825e3 100644--- a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx+++ b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx@@ -1,10 +1,12 @@import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";import {+ setupCancelJobRunEndpoint,setupGetTransformJobEndpoint,setupRunTransformJobEndpoint,} from "__support__/server-mocks";-import { renderWithProviders, screen } from "__support__/ui";+import { renderWithProviders, screen, waitFor } from "__support__/ui";import type { TransformJob } from "metabase-types/api";import {createMockTransformJob,@@ -81,6 +83,28 @@ describe("ScheduleSection", () => {).toBeInTheDocument();});+ it("should allow cancelling a running job", async () => {+ const job = createMockTransformJob({+ id: 3,+ last_run: createMockTransformRun({ id: 42, status: "started" }),+ });+ setupCancelJobRunEndpoint(3, 42);+ setup({ job });++ await userEvent.click(screen.getByTestId("cancel-button"));+ await userEvent.click(+ await screen.findByRole("button", { name: "Cancel run" }),+ );++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called(+ "path:/api/transform-job/3/runs/42/cancel",+ ),+ ).toBe(true);+ });+ });+it("should handle job run failures", async () => {const { rerender, onScheduleChange } = setup();const timestamp = new Date().toISOString();diff --git a/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx b/frontend/src/metabase/transforms/components/RunButton/RunButton.tsxindex 19ecad2798c..819b352dd73 100644--- a/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx+++ b/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx@@ -8,7 +8,14 @@ import {import { t } from "ttag";import { useSetting } from "metabase/common/hooks";-import { Button, type ButtonProps, Icon, Loader, Tooltip } from "metabase/ui";+import {+ Button,+ type ButtonProps,+ Icon,+ Loader,+ Menu,+ Tooltip,+} from "metabase/ui";import type { ColorName } from "metabase/ui/colors/types";import type {TransformId,@@ -16,6 +23,7 @@ import type {TransformRun,} from "metabase-types/api";+import { isActiveRunStatus } from "../../utils";import { LockedTransformsHoverCard } from "../LockedTransformsHoverCard/LockedTransformsHoverCard";const RECENT_TIMEOUT = 5000;@@ -29,6 +37,7 @@ type RunButtonProps = {size?: ButtonProps["size"];onRun: () => void;onCancel?: () => void;+ menuItems?: ReactNode;};export const RunButton = forwardRef(function RunButton(@@ -41,6 +50,7 @@ export const RunButton = forwardRef(function RunButton(size = "md",onRun,onCancel,+ menuItems,}: RunButtonProps,ref: Ref<HTMLButtonElement>,) {@@ -63,6 +73,9 @@ export const RunButton = forwardRef(function RunButton(setIsRecent(false);}, [id]);+ const isRunning = isActiveRunStatus(run?.status);+ const showMenu = menuItems != null && !isRunning;+const runButton = (<Button.Group><Button@@ -77,6 +90,24 @@ export const RunButton = forwardRef(function RunButton(>{label}</Button>+ {showMenu && (+ <Menu position="bottom-end">+ <Menu.Target>+ <Button+ variant="filled"+ color={color}+ disabled={isDisabled}+ size={size}+ px="sm"+ aria-label={t`More run options`}+ data-testid="run-options-button"+ >+ <Icon name="chevrondown" aria-hidden />+ </Button>+ </Menu.Target>+ <Menu.Dropdown>{menuItems}</Menu.Dropdown>+ </Menu>+ )}{allowCancellation && run?.status === "started" && (<Tooltip label={t`Cancel`}><Buttondiff --git a/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx b/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsxindex d92cc62fdaf..2549973b128 100644--- a/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx+++ b/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx@@ -1,21 +1,36 @@+import type { ReactNode } from "react";+import { mockSettings } from "__support__/settings";import { act, fireEvent, renderWithProviders, screen } from "__support__/ui";import { createMockState } from "metabase/redux/store/mocks";+import { Menu } from "metabase/ui";+import type { TransformRun } from "metabase-types/api";+import { createMockTransformRun } from "metabase-types/api/mocks";import { RunButton } from "./RunButton";function setup({isMeterLocked = null,+ run = null,+ menuItems,+ onRun = jest.fn(),}: {isMeterLocked?: boolean | null;+ run?: TransformRun | null;+ menuItems?: ReactNode;+ onRun?: () => void;} = {}) {jest.useFakeTimers();const state = createMockState({settings: mockSettings({ "transforms-meter-locked": isMeterLocked }),});- renderWithProviders(<RunButton id={1} run={null} onRun={jest.fn()} />, {- storeInitialState: state,- });+ renderWithProviders(+ <RunButton id={1} run={run} onRun={onRun} menuItems={menuItems} />,+ {+ storeInitialState: state,+ },+ );+ return { onRun };}describe("RunButton", () => {@@ -38,6 +53,32 @@ describe("RunButton", () => {});});+ it("calls onRun when the run button is clicked", () => {+ const { onRun } = setup();++ fireEvent.click(screen.getByTestId("run-button"));++ expect(onRun).toHaveBeenCalledTimes(1);+ });++ describe("with menu items", () => {+ const menuItems = <Menu.Item>{"Run upstream"}</Menu.Item>;++ it("shows the run options menu toggle when idle", () => {+ setup({ menuItems });++ expect(screen.getByTestId("run-options-button")).toBeInTheDocument();+ });++ it("hides the run options menu toggle while a run is in progress", () => {+ setup({ menuItems, run: createMockTransformRun({ status: "started" }) });++ expect(+ screen.queryByTestId("run-options-button"),+ ).not.toBeInTheDocument();+ });+ });+describe("when transforms are locked", () => {it("renders a disabled run button and shows an explanatory card on hover", async () => {setup({ isMeterLocked: true });diff --git a/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx b/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsxindex c9c5e19cdcf..100e8ca7ac2 100644--- a/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx+++ b/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx@@ -13,12 +13,13 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrimport { DataStudioBreadcrumbs } from "metabase/common/data-studio/components/DataStudioBreadcrumbs";import { PaneHeader } from "metabase/common/data-studio/components/PaneHeader";import { useSetting } from "metabase/common/hooks";-import { usePageTitle } from "metabase/hooks/use-page-title";import { useDispatch } from "metabase/redux";import { replace } from "metabase/router";+import { DetailedViewSwitch } from "metabase/transforms/components/DetailedViewSwitch";import { LockedTransformsBanner } from "metabase/transforms/components/LockedTransformsBanner/LockedTransformsBanner";import { POLLING_INTERVAL } from "metabase/transforms/constants";-import { Center, Flex, Stack } from "metabase/ui";+import { isActiveRunStatus } from "metabase/transforms/utils";+import { Center, Flex, Group, Stack } from "metabase/ui";import * as Urls from "metabase/urls";import type { TransformRun, TransformRunId } from "metabase-types/api";@@ -46,7 +47,6 @@ type RunListPageProps = {};export function RunListPage({ location }: RunListPageProps) {- usePageTitle(t`Runs`);const params = getParsedParams(location);const { page = 0 } = params;const { ref: containerRef, width: containerWidth } = useElementSize();@@ -173,14 +173,20 @@ export function RunListPage({ location }: RunListPageProps) {<LoadingAndErrorWrapper loading={isLoading} error={error} /></Center>) : (- <Stack flex="0 1 auto" mih={0} gap="lg">+ <Stack flex="0 1 auto" mih={0} gap="lg" pt="2.5rem">{isMeterLocked && <LockedTransformsBanner />}- <RunFilterBar- filterOptions={getFilterOptions(params)}- transforms={transforms}- tags={tags}- onFilterOptionsChange={handleFilterOptionsChange}- />+ <Group justify="space-between" align="center" wrap="nowrap">+ <RunFilterBar+ filterOptions={getFilterOptions(params)}+ transforms={transforms}+ tags={tags}+ onFilterOptionsChange={handleFilterOptionsChange}+ />+ <DetailedViewSwitch+ detailed={true}+ params={Urls.pickCommonRunListParams(params)}+ />+ </Group><RunTableruns={runs}tags={tags}@@ -212,7 +218,5 @@ export function RunListPage({ location }: RunListPageProps) {}export function isPollingNeeded(runs: TransformRun[] = []) {- return runs.some(- (run) => run.status === "started" || run.status === "canceling",- );+ return runs.some((run) => isActiveRunStatus(run.status));}diff --git a/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsx b/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsxnew file mode 100644index 00000000000..0caa006fa38--- /dev/null+++ b/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsx@@ -0,0 +1,15 @@+import { t } from "ttag";++import { usePageTitle } from "metabase/hooks/use-page-title";+import { Outlet } from "metabase/router";+import { Flex } from "metabase/ui";++export function RunsPage() {+ usePageTitle(t`Runs`);++ return (+ <Flex direction="column" w="100%" h="100%">+ <Outlet />+ </Flex>+ );+}diff --git a/frontend/src/metabase/transforms/pages/RunsPage/index.ts b/frontend/src/metabase/transforms/pages/RunsPage/index.tsnew file mode 100644index 00000000000..e27f52d10ad--- /dev/null+++ b/frontend/src/metabase/transforms/pages/RunsPage/index.ts@@ -0,0 +1 @@+export { RunsPage } from "./RunsPage";diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsxnew file mode 100644index 00000000000..19ca92f6376--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsx@@ -0,0 +1,97 @@+import { t } from "ttag";++import type { RelativeIntervalDirection } from "metabase/querying/common/types";+import { RunMethodFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/RunMethodFilterWidget";+import { StatusFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/StatusFilterWidget";+import { TimeFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/TimeFilterWidget";+import { TransformFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/TransformFilterWidget";+import { Group } from "metabase/ui";+import type {+ Transform,+ TransformGraphRunType,+ TransformId,+ TransformRunMethod,+ TransformRunStatus,+} from "metabase-types/api";++import type { TransformGraphRunFilterOptions } from "../types";++import { TypeFilterWidget } from "./TypeFilterWidget";++const PAST_INTERVAL_DIRECTIONS: RelativeIntervalDirection[] = [+ "past",+ "current",+];++type TransformGraphRunFilterBarProps = {+ filterOptions: TransformGraphRunFilterOptions;+ transforms: Transform[];+ onFilterOptionsChange: (+ filterOptions: TransformGraphRunFilterOptions,+ ) => void;+};++export function TransformGraphRunFilterBar({+ filterOptions,+ transforms,+ onFilterOptionsChange,+}: TransformGraphRunFilterBarProps) {+ const handleTypesChange = (types: TransformGraphRunType[]) => {+ onFilterOptionsChange({ ...filterOptions, types });+ };++ const handleTransformsChange = (transformIds: TransformId[]) => {+ onFilterOptionsChange({ ...filterOptions, transformIds });+ };++ const handleStatusesChange = (statuses: TransformRunStatus[]) => {+ onFilterOptionsChange({ ...filterOptions, statuses });+ };++ const handleStartTimeChange = (startTime: string | undefined) => {+ onFilterOptionsChange({ ...filterOptions, startTime });+ };++ const handleEndTimeChange = (endTime: string | undefined) => {+ onFilterOptionsChange({ ...filterOptions, endTime });+ };++ const handleRunMethodsChange = (runMethods: TransformRunMethod[]) => {+ onFilterOptionsChange({ ...filterOptions, runMethods });+ };++ return (+ <Group>+ <TypeFilterWidget+ types={filterOptions.types ?? []}+ onChange={handleTypesChange}+ />+ <TransformFilterWidget+ transformIds={filterOptions.transformIds ?? []}+ transforms={transforms}+ onChange={handleTransformsChange}+ />+ <TimeFilterWidget+ label={t`Started at`}+ value={filterOptions.startTime}+ availableDirections={PAST_INTERVAL_DIRECTIONS}+ onChange={handleStartTimeChange}+ />+ <TimeFilterWidget+ label={t`Ended at`}+ value={filterOptions.endTime}+ availableDirections={PAST_INTERVAL_DIRECTIONS}+ onChange={handleEndTimeChange}+ />+ <StatusFilterWidget+ label={t`Status`}+ statuses={filterOptions.statuses ?? []}+ onChange={handleStatusesChange}+ />+ <RunMethodFilterWidget+ runMethods={filterOptions.runMethods ?? []}+ onChange={handleRunMethodsChange}+ />+ </Group>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsxnew file mode 100644index 00000000000..4d4d43fcb92--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsx@@ -0,0 +1,124 @@+import { useDisclosure } from "@mantine/hooks";+import { type FormEvent, useState } from "react";+import { t } from "ttag";++import { FILTER_WIDGET_MIN_WIDTH } from "metabase/transforms/constants";+import { FilterFieldSet } from "metabase/transforms/pages/RunListPage/RunFilterBar/FilterFieldSet";+import {+ Box,+ Button,+ Checkbox,+ Divider,+ Group,+ Popover,+ Stack,+} from "metabase/ui";+import {+ TRANSFORM_GRAPH_RUN_TYPES,+ type TransformGraphRunType,+} from "metabase-types/api";++export function formatGraphRunType(type: TransformGraphRunType): string {+ switch (type) {+ case "job":+ return t`Job`;+ case "dag":+ return t`Upstream / downstream`;+ case "transform":+ return t`Single transformation`;+ }+}++function isTransformGraphRunType(+ value: string,+): value is TransformGraphRunType {+ return TRANSFORM_GRAPH_RUN_TYPES.some((type) => type === value);+}++type TypeFilterWidgetProps = {+ types: TransformGraphRunType[];+ onChange: (types: TransformGraphRunType[]) => void;+};++export function TypeFilterWidget({ types, onChange }: TypeFilterWidgetProps) {+ const [isOpened, { toggle, close }] = useDisclosure();++ const handleRemove = () => {+ onChange([]);+ };++ const handleSubmit = (nextTypes: TransformGraphRunType[]) => {+ close();+ onChange(nextTypes);+ };++ return (+ <Popover opened={isOpened} position="bottom-start" onDismiss={close}>+ <Popover.Target>+ <FilterFieldSet+ label={t`Type`}+ icon="list"+ displayValue={getDisplayValue(types)}+ onClick={toggle}+ onRemove={handleRemove}+ />+ </Popover.Target>+ <Popover.Dropdown>+ <TypeFilterForm types={types} onSubmit={handleSubmit} />+ </Popover.Dropdown>+ </Popover>+ );+}++function getDisplayValue(types: TransformGraphRunType[]) {+ if (types.length === 0) {+ return null;+ }+ return types.map(formatGraphRunType).join(", ");+}++type TypeFilterFormProps = {+ types: TransformGraphRunType[];+ onSubmit: (types: TransformGraphRunType[]) => void;+};++function TypeFilterForm({+ types: initialTypes,+ onSubmit,+}: TypeFilterFormProps) {+ const [types, setTypes] = useState(initialTypes);+ const isValid = types.length > 0;++ const handleChange = (values: string[]) => {+ setTypes(values.filter(isTransformGraphRunType));+ };++ const handleSubmit = (event: FormEvent) => {+ event.preventDefault();+ if (isValid) {+ onSubmit(types);+ }+ };++ return (+ <Box component="form" miw={FILTER_WIDGET_MIN_WIDTH} onSubmit={handleSubmit}>+ <Checkbox.Group value={types} onChange={handleChange}>+ <Stack p="md">+ {TRANSFORM_GRAPH_RUN_TYPES.map((type) => (+ <Checkbox+ key={type}+ value={type}+ label={formatGraphRunType(type)}+ />+ ))}+ </Stack>+ </Checkbox.Group>+ <Divider />+ <Group p="md" justify="end">+ <Button type="submit" variant="filled" disabled={!isValid}>+ {initialTypes.length > 0 ? t`Update filter` : t`Add filter`}+ </Button>+ </Group>+ </Box>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.tsnew file mode 100644index 00000000000..1062980da59--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.ts@@ -0,0 +1 @@+export { TransformGraphRunFilterBar } from "./TransformGraphRunFilterBar";diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.css b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.cssnew file mode 100644index 00000000000..5780d690fe1--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.css@@ -0,0 +1,7 @@+.main {+ overflow: hidden;+}++.resizing {+ user-select: none;+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsxnew file mode 100644index 00000000000..3f18761e0b7--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsx@@ -0,0 +1,209 @@+import { useDisclosure, useElementSize } from "@mantine/hooks";+import cx from "classnames";+import type { Location } from "history";+import { useCallback, useEffect, useMemo, useState } from "react";+import { t } from "ttag";++import {+ useListTransformGraphRunsQuery,+ useListTransformsQuery,+} from "metabase/api";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { PaginationControls } from "metabase/common/components/PaginationControls";+import { DataStudioBreadcrumbs } from "metabase/common/data-studio/components/DataStudioBreadcrumbs";+import { PaneHeader } from "metabase/common/data-studio/components/PaneHeader";+import { useDispatch } from "metabase/redux";+import { replace } from "metabase/router";+import { DetailedViewSwitch } from "metabase/transforms/components/DetailedViewSwitch";+import { POLLING_INTERVAL } from "metabase/transforms/constants";+import { isActiveRunStatus } from "metabase/transforms/utils";+import { Center, Flex, Group, Stack } from "metabase/ui";+import * as Urls from "metabase/urls";+import type { TransformGraphRun } from "metabase-types/api";++import { TransformGraphRunFilterBar } from "./TransformGraphRunFilterBar";+import S from "./TransformGraphRunListPage.module.css";+import { TransformGraphRunSidebar } from "./TransformGraphRunSidebar";+import { TransformGraphRunTable, getRowKey } from "./TransformGraphRunTable";+import { PAGE_SIZE } from "./constants";+import type {+ TransformGraphRunFilterOptions,+ TransformGraphRunSortOptions,+} from "./types";+import {+ getFilterOptions,+ getParsedParams,+ getSortOptions,+ hasFilterOptions,+} from "./utils";++const EMPTY_RUNS: TransformGraphRun[] = [];++type TransformGraphRunListPageProps = {+ location: Location;+};++export function TransformGraphRunListPage({+ location,+}: TransformGraphRunListPageProps) {+ const params = useMemo(() => getParsedParams(location), [location]);+ const filterOptions = useMemo(() => getFilterOptions(params), [params]);+ const { page = 0 } = params;+ const { ref: containerRef, width: containerWidth } = useElementSize();+ const [isResizing, { open: startResizing, close: stopResizing }] =+ useDisclosure();+ const [selectedRun, setSelectedRun] = useState<TransformGraphRun>();+ const [isPolling, setIsPolling] = useState(false);+ const dispatch = useDispatch();++ const {+ data,+ isLoading: isLoadingRuns,+ error,+ } = useListTransformGraphRunsQuery(+ {+ offset: page * PAGE_SIZE,+ limit: PAGE_SIZE,+ types: params.types,+ statuses: params.statuses,+ "transform-ids": params.transformIds,+ "start-time": params.startTime,+ "end-time": params.endTime,+ "run-methods": params.runMethods,+ "sort-column": params.sortColumn,+ "sort-direction": params.sortDirection,+ },+ {+ pollingInterval: isPolling ? POLLING_INTERVAL : undefined,+ },+ );++ const { data: transforms = [], isLoading: isLoadingTransforms } =+ useListTransformsQuery({});++ const isLoading = isLoadingRuns || isLoadingTransforms;++ if (isPolling !== isPollingNeeded(data?.data)) {+ setIsPolling(isPollingNeeded(data?.data));+ }++ const runs = data?.data ?? EMPTY_RUNS;++ useEffect(() => {+ setSelectedRun((current) =>+ current == null+ ? current+ : // Do not close sidebar on its own if selected run goes to next page+ (runs.find((run) => getRowKey(run) === getRowKey(current)) ??+ current),+ );+ }, [runs]);++ const handleParamsChange = useCallback(+ (newParams: Urls.TransformGraphRunListParams) => {+ dispatch(replace(Urls.transformGraphRunList(newParams)));+ },+ [dispatch],+ );++ const handleFilterOptionsChange = useCallback(+ (filterOptions: TransformGraphRunFilterOptions) => {+ handleParamsChange({ ...params, ...filterOptions, page: undefined });+ },+ [params, handleParamsChange],+ );++ const handleSortOptionsChange = useCallback(+ (sortOptions: TransformGraphRunSortOptions | undefined) => {+ handleParamsChange({+ ...params,+ sortColumn: sortOptions?.column,+ sortDirection: sortOptions?.direction,+ page: undefined,+ });+ },+ [params, handleParamsChange],+ );++ const handlePageChange = useCallback(+ (nextPage: number) => {+ handleParamsChange({ ...params, page: nextPage });+ },+ [params, handleParamsChange],+ );++ const handleSelect = useCallback(+ (run: TransformGraphRun) => setSelectedRun(run),+ [],+ );++ return (+ <Flex+ className={cx({ [S.resizing]: isResizing })}+ ref={containerRef}+ h="100%"+ wrap="nowrap"+ data-testid="transform-graph-run-list"+ >+ <Stack className={S.main} flex={1} px="3.5rem" pb="md" gap={0}>+ <PaneHeader+ breadcrumbs={<DataStudioBreadcrumbs>{t`Runs`}</DataStudioBreadcrumbs>}+ py={0}+ showMetabotButton+ />+ {isLoading || error != null ? (+ <Center h="100%">+ <LoadingAndErrorWrapper loading={isLoading} error={error} />+ </Center>+ ) : (+ <Stack flex="0 1 auto" mih={0} gap="lg" pt="2.5rem">+ <Group justify="space-between" align="center" wrap="nowrap">+ <TransformGraphRunFilterBar+ filterOptions={filterOptions}+ transforms={transforms}+ onFilterOptionsChange={handleFilterOptionsChange}+ />+ <DetailedViewSwitch+ detailed={false}+ params={Urls.pickCommonRunListParams(params)}+ />+ </Group>+ <TransformGraphRunTable+ runs={runs}+ hasFilters={hasFilterOptions(filterOptions)}+ sortOptions={getSortOptions(params)}+ onSortOptionsChange={handleSortOptionsChange}+ onSelect={handleSelect}+ />+ {data != null && data.total > PAGE_SIZE && (+ <Group justify="end">+ <PaginationControls+ page={page}+ pageSize={PAGE_SIZE}+ itemsLength={runs.length}+ total={data.total}+ showTotal+ onPreviousPage={() => handlePageChange(page - 1)}+ onNextPage={() => handlePageChange(page + 1)}+ />+ </Group>+ )}+ </Stack>+ )}+ </Stack>+ {selectedRun != null && (+ <TransformGraphRunSidebar+ run={selectedRun}+ containerWidth={containerWidth}+ onResizeStart={startResizing}+ onResizeStop={stopResizing}+ onClose={() => setSelectedRun(undefined)}+ />+ )}+ </Flex>+ );+}++export function isPollingNeeded(runs: TransformGraphRun[] = []) {+ return runs.some((run) => isActiveRunStatus(run.status));+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsxnew file mode 100644index 00000000000..b56c89d6ad3--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsx@@ -0,0 +1,264 @@+import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";++import {+ setupCancelJobRunEndpoint,+ setupListDagRunTransformRunsEndpoint,+ setupListJobRunTransformRunsEndpoint,+ setupListTransformGraphRunsEndpoint,+ setupListTransformsEndpoint,+ setupUserMetabotPermissionsEndpoint,+} from "__support__/server-mocks";+import {+ mockGetBoundingClientRect,+ renderWithProviders,+ screen,+ waitFor,+ within,+} from "__support__/ui";+import { Route, withRouteProps } from "metabase/router";+import type { TransformGraphRun } from "metabase-types/api";+import {+ createMockListTransformGraphRunsResponse,+ createMockTransformGraphRun,+ createMockTransformRunForJobRun,+} from "metabase-types/api/mocks";++import { TransformGraphRunListPage } from "./TransformGraphRunListPage";++const RoutedTransformGraphRunListPage = withRouteProps(+ TransformGraphRunListPage,+);++type SetupOpts = {+ runs?: TransformGraphRun[];+ initialRoute?: string;+};++function setup({+ runs = [],+ initialRoute = "/data-studio/transforms/runs",+}: SetupOpts = {}) {+ setupUserMetabotPermissionsEndpoint();+ setupListTransformsEndpoint([]);+ setupListTransformGraphRunsEndpoint(+ createMockListTransformGraphRunsResponse({+ data: runs,+ total: runs.length,+ }),+ );+ mockGetBoundingClientRect({ width: 1200, height: 800 });++ const { history } = renderWithProviders(+ <Route+ path="/data-studio/transforms/runs"+ element={<RoutedTransformGraphRunListPage />}+ />,+ { withRouter: true, initialRoute },+ );++ return { history };+}++const JOB_RUN = createMockTransformGraphRun({+ run_type: "job",+ id: 101,+ entity_id: 1,+ name: "Hourly refresh",+ run_method: "cron",+});++const DAG_RUN = createMockTransformGraphRun({+ run_type: "dag",+ id: 202,+ entity_id: 12,+ name: "Customers deduped",+ direction: "upstream",+ transform_count: 5,+ status: "failed",+});++const TRANSFORM_RUN = createMockTransformGraphRun({+ run_type: "transform",+ id: 301,+ entity_id: 20,+ name: "Products normalized",+});++describe("TransformGraphRunListPage", () => {+ it("renders job, DAG, and standalone runs with a Run name and Type", async () => {+ setup({ runs: [JOB_RUN, DAG_RUN, TRANSFORM_RUN] });++ expect(await screen.findByText("Hourly refresh")).toBeInTheDocument();+ expect(screen.getByText("4 dependencies")).toBeInTheDocument();+ expect(screen.getByText("Customers deduped")).toBeInTheDocument();+ expect(screen.getByText("Products normalized")).toBeInTheDocument();++ const table = screen.getByRole("treegrid", { name: "Runs" });+ expect(within(table).getByText("Job")).toBeInTheDocument();+ expect(within(table).getAllByText("Transformation")).toHaveLength(2);+ });++ it("shows the Detailed view switch unchecked in the grouped view", async () => {+ setup({ runs: [] });+ expect(await screen.findByLabelText("Detailed view")).not.toBeChecked();+ });++ it("preserves shared filters (and drops view-specific params) when toggling to the detailed view", async () => {+ const { history } = setup({+ runs: [],+ initialRoute:+ "/data-studio/transforms/runs?statuses=failed&transform-ids=7&start-time=2024-01-01&run-methods=cron&sort-column=start_time&sort-direction=desc&page=2",+ });++ await userEvent.click(await screen.findByLabelText("Detailed view"));++ const location = history?.getCurrentLocation();+ expect(location?.pathname).toBe("/data-studio/transforms/runs/individual");++ const search = new URLSearchParams(location?.search);+ expect(search.getAll("statuses")).toEqual(["failed"]);+ expect(search.getAll("transform-ids")).toEqual(["7"]);+ expect(search.get("start-time")).toBe("2024-01-01");+ expect(search.getAll("run-methods")).toEqual(["cron"]);++ expect(search.has("sort-column")).toBe(false);+ expect(search.has("page")).toBe(false);+ });++ it("greys a deleted-entity run and hides the Transforms section for a deleted job", async () => {+ const deletedJobRun = createMockTransformGraphRun({+ run_type: "job",+ id: 103,+ entity_id: null,+ name: "Removed job",+ run_method: "cron",+ });+ setup({ runs: [deletedJobRun] });++ const name = await screen.findByText("Removed job");+ await userEvent.hover(name);+ expect(+ await screen.findByText("Removed job has been deleted"),+ ).toBeInTheDocument();++ await userEvent.click(name);+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");++ expect(within(sidebar).queryByText("Transforms")).not.toBeInTheDocument();+ expect(+ within(sidebar).queryByRole("link", { name: "View this job" }),+ ).not.toBeInTheDocument();+ });++ it("still shows member transforms for a DAG run whose seed transform was deleted", async () => {+ const deletedDagRun = createMockTransformGraphRun({+ run_type: "dag",+ id: 205,+ entity_id: null,+ name: "Removed seed",+ direction: "upstream",+ transform_count: 3,+ status: "succeeded",+ });+ setupListDagRunTransformRunsEndpoint(205, [+ createMockTransformRunForJobRun({+ id: 1,+ transform_name: "Orders cleaned",+ status: "succeeded",+ }),+ ]);+ setup({ runs: [deletedDagRun] });++ await userEvent.click(await screen.findByText("Removed seed"));++ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");++ expect(within(sidebar).getByText("Transforms")).toBeInTheDocument();+ expect(+ await within(sidebar).findByText("Orders cleaned"),+ ).toBeInTheDocument();+ });++ it("drills into a DAG run's member transforms via the dag-run endpoint", async () => {+ setupListDagRunTransformRunsEndpoint(202, [+ createMockTransformRunForJobRun({+ id: 1,+ transform_name: "Orders cleaned",+ status: "succeeded",+ }),+ createMockTransformRunForJobRun({+ id: 2,+ transform_name: "Customers deduped",+ status: "failed",+ message: "boom",+ }),+ ]);+ setup({ runs: [DAG_RUN] });++ await userEvent.click(await screen.findByText("Customers deduped"));++ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");+ expect(+ await within(sidebar).findByText("Orders cleaned"),+ ).toBeInTheDocument();++ expect(+ within(sidebar).getByRole("region", { name: "Error" }),+ ).toBeInTheDocument();++ expect(+ within(sidebar).getByRole("link", { name: "View this transform" }),+ ).toHaveAttribute("href", "/data-studio/transforms/12");+ });++ it("shows a standalone transform run as its own single member (no fetch)", async () => {+ setup({ runs: [TRANSFORM_RUN] });++ await userEvent.click(await screen.findByText("Products normalized"));++ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");+ const member = within(sidebar).getByTestId("transform-run-item");+ expect(member).toHaveTextContent("Products normalized");+ });++ it("cancels an in-progress job run via the job-run cancel endpoint", async () => {+ const startedJob = createMockTransformGraphRun({+ run_type: "job",+ id: 102,+ entity_id: 2,+ name: "Nightly rollups",+ run_method: "cron",+ status: "started",+ });+ setupListJobRunTransformRunsEndpoint(2, 102, [+ createMockTransformRunForJobRun({ id: 5, transform_name: "Step" }),+ ]);+ setupCancelJobRunEndpoint(2, 102);+ setup({ runs: [startedJob] });++ await userEvent.click(await screen.findByText("Nightly rollups"));++ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");+ expect(+ within(sidebar).getByRole("link", { name: "View this job" }),+ ).toHaveAttribute("href", "/data-studio/transforms/jobs/2");++ await userEvent.click(+ within(sidebar).getByRole("button", { name: "Cancel run" }),+ );++ const dialog = await screen.findByRole("dialog");+ await userEvent.click(+ within(dialog).getByRole("button", { name: "Cancel run" }),+ );++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called(+ "path:/api/transform-job/2/runs/102/cancel",+ ),+ ).toBe(true);+ });+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsxnew file mode 100644index 00000000000..0a76dbd9726--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsx@@ -0,0 +1,55 @@+import { t } from "ttag";++import { DateTime } from "metabase/common/components/DateTime";+import { SidebarInfoRow } from "metabase/transforms/components/SidebarInfoRow";+import {+ formatRunMethod,+ formatStatus,+ getRunDurationMs,+ isErrorStatus,+} from "metabase/transforms/utils";+import { Box, Card } from "metabase/ui";+import { EMPTY_CELL_PLACEHOLDER } from "metabase/utils/constants";+import { formatDurationLong } from "metabase/utils/formatting/time";+import type { TransformGraphRun } from "metabase-types/api";++import { formatRunType } from "../TransformGraphRunTable";++type TransformGraphRunInfoSectionProps = {+ run: TransformGraphRun;+};++export function TransformGraphRunInfoSection({+ run,+}: TransformGraphRunInfoSectionProps) {+ const durationMs = getRunDurationMs(run);++ return (+ <Card p={0} shadow="none" withBorder role="region" aria-label={t`Info`}>+ <SidebarInfoRow label={t`Type`}>{formatRunType(run)}</SidebarInfoRow>+ <SidebarInfoRow label={t`Status`}>+ <Box c={isErrorStatus(run.status) ? "feedback-negative" : undefined}>+ {formatStatus(run.status)}+ </Box>+ </SidebarInfoRow>+ <SidebarInfoRow label={t`Trigger`}>+ {run.run_method != null+ ? formatRunMethod(run.run_method)+ : EMPTY_CELL_PLACEHOLDER}+ </SidebarInfoRow>+ <SidebarInfoRow label={t`Started at`}>+ <DateTime value={run.start_time} unit="minute" />+ </SidebarInfoRow>+ {run.end_time != null && (+ <SidebarInfoRow label={t`Ended at`}>+ <DateTime value={run.end_time} unit="minute" />+ </SidebarInfoRow>+ )}+ <SidebarInfoRow label={t`Duration`}>+ {durationMs != null+ ? formatDurationLong(durationMs)+ : EMPTY_CELL_PLACEHOLDER}+ </SidebarInfoRow>+ </Card>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.css b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.cssnew file mode 100644index 00000000000..ce06f4b9e89--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.css@@ -0,0 +1,12 @@+.sidebar {+ border-left: 1px solid var(--mb-color-border-neutral);+}++.header {+ flex-shrink: 0;+ border-bottom: 1px solid var(--mb-color-border-neutral);+}++.scrollArea {+ overflow-y: auto;+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsxnew file mode 100644index 00000000000..50bcd198fef--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsx@@ -0,0 +1,342 @@+import { useDisclosure } from "@mantine/hooks";+import { memo, useMemo, useState } from "react";+import { match } from "ts-pattern";+import { t } from "ttag";++import {+ skipToken,+ useCancelCurrentTransformRunMutation,+ useCancelDagRunMutation,+ useCancelJobRunMutation,+ useListDagRunTransformRunsQuery,+ useListJobRunTransformRunsQuery,+} from "metabase/api";+import { ConfirmModal } from "metabase/common/components/ConfirmModal";+import { ForwardRefLink } from "metabase/common/components/Link";+import { ListEmptyState } from "metabase/common/components/ListEmptyState";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { useMetadataToasts } from "metabase/metadata/hooks";+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";+import { SidebarResizableBox } from "metabase/transforms/components/SidebarResizableBox";+import { POLLING_INTERVAL } from "metabase/transforms/constants";+import { isActiveRunStatus } from "metabase/transforms/utils";+import {+ ActionIcon,+ Badge,+ Box,+ Button,+ Card,+ FixedSizeIcon,+ Flex,+ Group,+ Stack,+ Title,+ Tooltip,+} from "metabase/ui";+import * as Urls from "metabase/urls";+import { isResourceNotFoundError } from "metabase/utils/errors";+import type {+ TransformGraphRun,+ TransformRunForJobRun,+} from "metabase-types/api";++import { TransformRunItem } from "../../JobRunListPage/JobRunSidebar/TransformRunItem";+import { RunName, isDeletedRun } from "../TransformGraphRunTable";++import { TransformGraphRunInfoSection } from "./TransformGraphRunInfoSection";+import S from "./TransformGraphRunSidebar.module.css";++const EMPTY_TRANSFORM_RUNS: TransformRunForJobRun[] = [];++function canCancelRun(run: TransformGraphRun): boolean {+ if (run.status !== "started") {+ return false;+ }+ return run.run_type === "dag" || run.entity_id != null;+}++type TransformGraphRunSidebarProps = {+ run: TransformGraphRun;+ containerWidth: number;+ onResizeStart: () => void;+ onResizeStop: () => void;+ onClose: () => void;+};++export const TransformGraphRunSidebar = memo(function TransformGraphRunSidebar({+ run,+ containerWidth,+ onResizeStart,+ onResizeStop,+ onClose,+}: TransformGraphRunSidebarProps) {+ const [isPolling, setIsPolling] = useState(false);+ const pollingInterval = isPolling ? POLLING_INTERVAL : undefined;++ const areMembersUnavailable = run.run_type === "job" && isDeletedRun(run);++ const dagResult = useListDagRunTransformRunsQuery(+ run.run_type === "dag" ? { dagRunId: run.id } : skipToken,+ { pollingInterval },+ );+ const jobResult = useListJobRunTransformRunsQuery(+ run.run_type === "job" && run.entity_id != null+ ? { jobId: run.entity_id, runId: run.id }+ : skipToken,+ { pollingInterval },+ );+ const standaloneRuns = useMemo<TransformRunForJobRun[]>(+ () =>+ run.run_type === "transform"+ ? [+ {+ id: run.id,+ transform_id: run.entity_id,+ transform_name: run.name,+ job_run_id: null,+ status: run.status,+ run_method: run.run_method ?? "manual",+ start_time: run.start_time,+ end_time: run.end_time,+ message: run.message,+ },+ ]+ : EMPTY_TRANSFORM_RUNS,+ [run],+ );++ const { transformRuns, isLoading, error } = match(run.run_type)+ .with("dag", () => ({+ transformRuns: dagResult.data ?? EMPTY_TRANSFORM_RUNS,+ isLoading: dagResult.isLoading,+ error: dagResult.error,+ }))+ .with("job", () => ({+ transformRuns: jobResult.data ?? EMPTY_TRANSFORM_RUNS,+ isLoading: jobResult.isLoading,+ error: jobResult.error,+ }))+ .with("transform", () => ({+ transformRuns: standaloneRuns,+ isLoading: false,+ error: undefined,+ }))+ .exhaustive();++ const shouldPoll =+ isActiveRunStatus(run.status) ||+ transformRuns.some((transformRun) =>+ isActiveRunStatus(transformRun.status),+ );+ if (isPolling !== shouldPoll) {+ setIsPolling(shouldPoll);+ }++ return (+ <SidebarResizableBox+ containerWidth={containerWidth}+ onResizeStart={onResizeStart}+ onResizeStop={onResizeStop}+ >+ <Flex+ className={S.sidebar}+ direction="column"+ flex={1}+ h="100%"+ bg="background_page-primary"+ data-testid="transform-graph-run-sidebar"+ >+ <Box className={S.header} p="lg">+ <TransformGraphRunSidebarHeader run={run} onClose={onClose} />+ </Box>+ <Box className={S.scrollArea} flex={1} mih={0} p="lg">+ <Stack gap="xl">+ <Stack gap="md">+ <TransformGraphRunInfoSection run={run} />+ {canCancelRun(run) && <CancelationSection run={run} />}+ </Stack>+ {!areMembersUnavailable && (+ <Stack gap="md">+ <Group gap="sm" wrap="nowrap">+ <Badge variant="filled" bg="core-brand">+ {transformRuns.length}+ </Badge>+ <Title order={5}>{t`Transforms`}</Title>+ </Group>+ <TransformRunList+ transformRuns={transformRuns}+ isLoading={isLoading}+ error={error}+ />+ </Stack>+ )}+ </Stack>+ </Box>+ </Flex>+ </SidebarResizableBox>+ );+});++type CancelationSectionProps = {+ run: TransformGraphRun;+};++function CancelationSection({ run }: CancelationSectionProps) {+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();+ const { sendErrorToast } = useMetadataToasts();+ const [cancelDagRun, { isLoading: isCancelingDag }] =+ useCancelDagRunMutation();+ const [cancelJobRun, { isLoading: isCancelingJob }] =+ useCancelJobRunMutation();+ const [cancelTransformRun, { isLoading: isCancelingTransform }] =+ useCancelCurrentTransformRunMutation();+ const isCanceling = isCancelingDag || isCancelingJob || isCancelingTransform;++ const handleCancel = async () => {+ let error: unknown;+ if (run.run_type === "dag") {+ ({ error } = await cancelDagRun(run.id));+ } else if (run.run_type === "job" && run.entity_id != null) {+ ({ error } = await cancelJobRun({ jobId: run.entity_id, runId: run.id }));+ } else if (run.run_type === "transform" && run.entity_id != null) {+ ({ error } = await cancelTransformRun(run.entity_id));+ }+ closeModal();+ if (error && !isResourceNotFoundError(error)) {+ sendErrorToast(t`Failed to cancel run`);+ }+ };++ return (+ <>+ <Group justify="flex-end">+ <Button+ variant="filled"+ color="feedback-negative"+ disabled={isCanceling}+ onClick={openModal}+ >+ {t`Cancel run`}+ </Button>+ </Group>+ <ConfirmModal+ title={t`Cancel this run?`}+ message={t`This stops the run and requests cancellation of any transforms still in progress. Transforms that have already finished won't be reverted.`}+ confirmButtonText={t`Cancel run`}+ closeButtonText={t`Keep running`}+ opened={isModalOpen}+ onClose={closeModal}+ onConfirm={handleCancel}+ />+ </>+ );+}++type TransformRunListProps = {+ transformRuns: TransformRunForJobRun[];+ isLoading: boolean;+ error: unknown;+};++function TransformRunList({+ transformRuns,+ isLoading,+ error,+}: TransformRunListProps) {+ if (isLoading || error != null) {+ return <LoadingAndErrorWrapper loading={isLoading} error={error} />;+ }++ if (transformRuns.length === 0) {+ return <ListEmptyState label={t`No transform runs`} />;+ }++ return (+ <Card p={0} shadow="none" withBorder>+ {transformRuns.map((transformRun) => (+ <TransformRunItem key={transformRun.id} transformRun={transformRun} />+ ))}+ </Card>+ );+}++type TransformGraphRunSidebarHeaderProps = {+ run: TransformGraphRun;+ onClose: () => void;+};++function TransformGraphRunSidebarHeader({+ run,+ onClose,+}: TransformGraphRunSidebarHeaderProps) {+ return (+ <Group+ justify="space-between"+ align="center"+ wrap="nowrap"+ gap="sm"+ data-testid="transform-graph-run-sidebar-header"+ >+ <Title order={3}>+ <RunName run={run} gap="sm" />+ </Title>+ <Group gap="xs" wrap="nowrap">+ <HeaderEntityActions run={run} />+ <ActionIcon aria-label={t`Close`} onClick={onClose}>+ <FixedSizeIcon name="close" />+ </ActionIcon>+ </Group>+ </Group>+ );+}++function HeaderEntityActions({ run }: { run: TransformGraphRun }) {+ const { entity_id } = run;+ if (entity_id == null) {+ return null;+ }++ if (run.run_type === "job") {+ return (+ <Tooltip label={t`View this job`}>+ <ActionIcon+ component={ForwardRefLink}+ to={Urls.transformJob(entity_id)}+ target="_blank"+ aria-label={t`View this job`}+ >+ <FixedSizeIcon name="external" />+ </ActionIcon>+ </Tooltip>+ );+ }++ return (+ <>+ <Tooltip label={t`View this transform`}>+ <ActionIcon+ component={ForwardRefLink}+ to={Urls.transform(entity_id)}+ target="_blank"+ aria-label={t`View this transform`}+ >+ <FixedSizeIcon name="external" />+ </ActionIcon>+ </Tooltip>+ {PLUGIN_DEPENDENCIES.isEnabled && (+ <Tooltip label={t`View in dependency graph`}>+ <ActionIcon+ component={ForwardRefLink}+ to={Urls.dependencyGraph({+ entry: { id: entity_id, type: "transform" },+ })}+ target="_blank"+ aria-label={t`View in dependency graph`}+ >+ <FixedSizeIcon name="dependencies" />+ </ActionIcon>+ </Tooltip>+ )}+ </>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.tsnew file mode 100644index 00000000000..5b56255155c--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.ts@@ -0,0 +1 @@+export { TransformGraphRunSidebar } from "./TransformGraphRunSidebar";diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsxnew file mode 100644index 00000000000..be317d53ca0--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsx@@ -0,0 +1,89 @@+import type { Row, SortingState, Updater } from "@tanstack/react-table";+import { useCallback, useMemo } from "react";+import { t } from "ttag";++import { ListEmptyState } from "metabase/common/components/ListEmptyState";+import { useSetting } from "metabase/common/hooks";+import { Card, TreeTable, useTreeTableInstance } from "metabase/ui";+import type { TransformGraphRun } from "metabase-types/api";++import type { TransformGraphRunSortOptions } from "../types";++import {+ getColumns,+ getRowKey,+ getSortingOptions,+ getSortingState,+} from "./utils";++type TransformGraphRunTableProps = {+ runs: TransformGraphRun[];+ hasFilters: boolean;+ sortOptions: TransformGraphRunSortOptions | undefined;+ onSortOptionsChange: (+ sortOptions: TransformGraphRunSortOptions | undefined,+ ) => void;+ onSelect: (run: TransformGraphRun) => void;+};++export function TransformGraphRunTable({+ runs,+ hasFilters,+ sortOptions,+ onSortOptionsChange,+ onSelect,+}: TransformGraphRunTableProps) {+ const systemTimezone = useSetting("system-timezone");++ const columns = useMemo(() => getColumns(systemTimezone), [systemTimezone]);++ const sortingState = useMemo(+ () => getSortingState(sortOptions),+ [sortOptions],+ );++ const notFoundLabel = hasFilters ? t`No runs found` : t`No runs yet`;++ const handleRowClick = useCallback(+ (row: Row<TransformGraphRun>) => {+ onSelect(row.original);+ },+ [onSelect],+ );++ const handleSortingChange = useCallback(+ (updater: Updater<SortingState>) => {+ const newSortingState =+ typeof updater === "function" ? updater(sortingState) : updater;+ onSortOptionsChange(getSortingOptions(newSortingState));+ },+ [sortingState, onSortOptionsChange],+ );++ const treeTableInstance = useTreeTableInstance<TransformGraphRun>({+ data: runs,+ columns,+ sorting: sortingState,+ manualSorting: true,+ getNodeId: getRowKey,+ onSortingChange: handleSortingChange,+ });++ return (+ <Card+ style={{ overflow: "hidden" }}+ p={0}+ flex="0 1 auto"+ mih={0}+ shadow="none"+ withBorder+ >+ <TreeTable+ instance={treeTableInstance}+ emptyState={<ListEmptyState label={notFoundLabel} />}+ ariaLabel={t`Runs`}+ onRowClick={handleRowClick}+ />+ </Card>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.tsnew file mode 100644index 00000000000..86ff3029a81--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.ts@@ -0,0 +1,8 @@+export { TransformGraphRunTable } from "./TransformGraphRunTable";+export {+ getRowKey,+ formatRunName,+ RunName,+ formatRunType,+ isDeletedRun,+} from "./utils";diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsxnew file mode 100644index 00000000000..c6e2407db2b--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsx@@ -0,0 +1,306 @@+import type { SortingState } from "@tanstack/react-table";+import { msgid, ngettext, t } from "ttag";++import { TimezoneIndicator } from "metabase/transforms/components/TimezoneIndicator";+import {+ formatRunMethod,+ formatStatus,+ getRunDurationMs,+ isErrorStatus,+ parseTimestampWithTimezone,+} from "metabase/transforms/utils";+import type { MantineSize, TreeTableColumnDef } from "metabase/ui";+import {+ Box,+ Ellipsified,+ Flex,+ Group,+ Icon,+ SortableHeaderPill,+ Text,+ Tooltip,+} from "metabase/ui";+import { EMPTY_CELL_PLACEHOLDER } from "metabase/utils/constants";+import { formatDurationLong } from "metabase/utils/formatting/time";+import {+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,+ type TransformGraphRun,+ type TransformGraphRunSortColumn,+} from "metabase-types/api";++import type { TransformGraphRunSortOptions } from "../types";++export function getRowKey(run: TransformGraphRun): string {+ return `${run.run_type}-${run.id}`;+}++export function isDeletedRun(run: TransformGraphRun): boolean {+ return run.entity_id == null;+}++function dagDependencyCount(run: TransformGraphRun): number | null {+ if (run.run_type !== "dag" || run.transform_count == null) {+ return null;+ }+ const count = run.transform_count - 1; // Doesn't include transform that triggered this DAG run+ return count > 0 ? count : null;+}++function dependenciesText(count: number): string {+ return ngettext(msgid`${count} dependency`, `${count} dependencies`, count);+}++function dependentsText(count: number): string {+ return ngettext(msgid`${count} dependent`, `${count} dependents`, count);+}++export function formatRunName(run: TransformGraphRun): string {+ const name = run.name ?? t`Deleted`;+ const count = dagDependencyCount(run);+ if (count == null) {+ return name;+ }+ return run.direction === "upstream"+ ? `${dependenciesText(count)} › ${name}`+ : `${name} › ${dependentsText(count)}`;+}++function RunEntityName({ run }: { run: TransformGraphRun }) {+ const name = run.name ?? t`Deleted`;+ if (!isDeletedRun(run)) {+ return <Ellipsified fw="bold">{name}</Ellipsified>;+ }+ return (+ <Ellipsified>+ <Tooltip label={t`${name} has been deleted`}>+ <Text c="text-disabled" component="span" display="inline" fs="italic">+ {name}+ </Text>+ </Tooltip>+ </Ellipsified>+ );+}++type RunNameProps = {+ run: TransformGraphRun;+ gap?: MantineSize;+};++export function RunName({ run, gap = "xs" }: RunNameProps) {+ const count = dagDependencyCount(run);+ if (count == null) {+ return <RunEntityName run={run} />;+ }+ const chevron = (+ <Icon+ name="chevronright"+ c="icon-secondary"+ size={8}+ mt="0.15em"+ aria-hidden+ />+ );+ const countText = (+ <Text+ component="span"+ c="text-secondary"+ fz="inherit"+ lh="inherit"+ fw="normal"+ style={{ whiteSpace: "nowrap" }}+ >+ {run.direction === "upstream"+ ? dependenciesText(count)+ : dependentsText(count)}+ </Text>+ );+ return (+ <Flex+ component="span"+ display="inline-flex"+ align="center"+ gap={gap}+ miw={0}+ >+ {run.direction === "upstream" ? (+ <>+ {countText}+ {chevron}+ <RunEntityName run={run} />+ </>+ ) : (+ <>+ <RunEntityName run={run} />+ {chevron}+ {countText}+ </>+ )}+ </Flex>+ );+}++export function formatRunType(run: TransformGraphRun): string {+ return run.run_type === "job" ? t`Job` : t`Transformation`;+}++function getRunColumn(): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "run",+ header: t`Run`,+ width: 320,+ accessorFn: (run) => formatRunName(run),+ cell: ({ row }) => <RunName run={row.original} />,+ };+}++function getTypeColumn(): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "type",+ header: t`Type`,+ width: 140,+ accessorFn: (run) => formatRunType(run),+ cell: ({ getValue }) => <Ellipsified>{String(getValue())}</Ellipsified>,+ };+}++function getStatusColumn(): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "status",+ header: t`Status`,+ width: 120,+ accessorFn: (row) => formatStatus(row.status),+ cell: ({ row }) => {+ const { status } = row.original;+ return (+ <Box c={isErrorStatus(status) ? "feedback-negative" : undefined}>+ {formatStatus(status)}+ </Box>+ );+ },+ };+}++function getRunMethodColumn(): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "run-method",+ header: t`Trigger`,+ width: 120,+ accessorFn: (row) =>+ row.run_method != null ? formatRunMethod(row.run_method) : null,+ cell: ({ getValue }) => {+ const value = getValue<string | null>();+ if (value == null) {+ return EMPTY_CELL_PLACEHOLDER;+ }+ return <Ellipsified>{value}</Ellipsified>;+ },+ };+}++function getStartedAtColumn(+ systemTimezone: string | undefined,+): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "start_time" satisfies TransformGraphRunSortColumn,+ header: ({ header }) => (+ <Group gap="xs" wrap="nowrap">+ <SortableHeaderPill+ name={t`Started at`}+ sort={header.column.getIsSorted() || undefined}+ />+ <TimezoneIndicator />+ </Group>+ ),+ width: "auto",+ enableSorting: true,+ accessorFn: (run) =>+ parseTimestampWithTimezone(run.start_time, systemTimezone).format("lll"),+ cell: ({ getValue }) => <Ellipsified>{String(getValue())}</Ellipsified>,+ };+}++function getEndedAtColumn(+ systemTimezone: string | undefined,+): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "end_time" satisfies TransformGraphRunSortColumn,+ header: ({ header }) => (+ <Group gap="xs" wrap="nowrap">+ <SortableHeaderPill+ name={t`Ended at`}+ sort={header.column.getIsSorted() || undefined}+ />+ <TimezoneIndicator />+ </Group>+ ),+ width: "auto",+ enableSorting: true,+ accessorFn: (run) =>+ run.end_time != null+ ? parseTimestampWithTimezone(run.end_time, systemTimezone).format("lll")+ : null,+ cell: ({ getValue }) => {+ const value = getValue<string | null>();+ if (value == null) {+ return EMPTY_CELL_PLACEHOLDER;+ }+ return <Ellipsified>{value}</Ellipsified>;+ },+ };+}++function getDurationColumn(): TreeTableColumnDef<TransformGraphRun> {+ return {+ id: "duration",+ header: t`Duration`,+ width: 120,+ accessorFn: (run) => getRunDurationMs(run),+ cell: ({ getValue }) => {+ const ms = getValue<number | null>();+ if (ms == null) {+ return EMPTY_CELL_PLACEHOLDER;+ }+ return <Ellipsified>{formatDurationLong(ms)}</Ellipsified>;+ },+ };+}++export function getColumns(+ systemTimezone: string | undefined,+): TreeTableColumnDef<TransformGraphRun>[] {+ return [+ getRunColumn(),+ getTypeColumn(),+ getStartedAtColumn(systemTimezone),+ getEndedAtColumn(systemTimezone),+ getDurationColumn(),+ getStatusColumn(),+ getRunMethodColumn(),+ ];+}++export function getSortingState(+ sortOptions: TransformGraphRunSortOptions | undefined,+): SortingState {+ return sortOptions != null+ ? [{ id: sortOptions.column, desc: sortOptions.direction === "desc" }]+ : [];+}++export function getSortingOptions(+ sortingState: SortingState,+): TransformGraphRunSortOptions | undefined {+ if (sortingState.length === 0) {+ return undefined;+ }++ const { id, desc } = sortingState[0];+ const column = TRANSFORM_GRAPH_RUN_SORT_COLUMNS.find(+ (column) => column === id,+ );+ if (column == null) {+ return undefined;+ }++ return { column, direction: desc ? "desc" : "asc" };+}diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.tsnew file mode 100644index 00000000000..45d559e788d--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.ts@@ -0,0 +1 @@+export const PAGE_SIZE = 25;diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.tsnew file mode 100644index 00000000000..4e598a5e24a--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.ts@@ -0,0 +1 @@+export { TransformGraphRunListPage } from "./TransformGraphRunListPage";diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.tsnew file mode 100644index 00000000000..1960bb6aa5c--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.ts@@ -0,0 +1,22 @@+import type {+ SortDirection,+ TransformGraphRunSortColumn,+ TransformGraphRunType,+ TransformId,+ TransformRunMethod,+ TransformRunStatus,+} from "metabase-types/api";++export type TransformGraphRunFilterOptions = {+ types?: TransformGraphRunType[];+ statuses?: TransformRunStatus[];+ transformIds?: TransformId[];+ startTime?: string;+ endTime?: string;+ runMethods?: TransformRunMethod[];+};++export type TransformGraphRunSortOptions = {+ column: TransformGraphRunSortColumn;+ direction: SortDirection;+};diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.tsnew file mode 100644index 00000000000..70de18c9d42--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.ts@@ -0,0 +1,86 @@+import type { Location } from "history";++import * as Urls from "metabase/urls";+import {+ SORT_DIRECTIONS,+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,+ TRANSFORM_GRAPH_RUN_TYPES,+ TRANSFORM_RUN_METHODS,+ TRANSFORM_RUN_STATUSES,+} from "metabase-types/api";++import type {+ TransformGraphRunFilterOptions,+ TransformGraphRunSortOptions,+} from "./types";++export function getParsedParams(+ location: Location,+): Urls.TransformGraphRunListParams {+ const {+ page,+ types,+ statuses,+ "transform-ids": transformIds,+ "start-time": startTime,+ "end-time": endTime,+ "run-methods": runMethods,+ "sort-column": sortColumn,+ "sort-direction": sortDirection,+ } = location.query;++ return {+ page: Urls.parseNumberParam(page),+ types: Urls.parseListParam(types, (v) =>+ Urls.parseEnumParam(v, TRANSFORM_GRAPH_RUN_TYPES),+ ),+ statuses: Urls.parseListParam(statuses, (v) =>+ Urls.parseEnumParam(v, TRANSFORM_RUN_STATUSES),+ ),+ transformIds: Urls.parseListParam(transformIds, Urls.parseNumberParam),+ startTime: Urls.parseStringParam(startTime),+ endTime: Urls.parseStringParam(endTime),+ runMethods: Urls.parseListParam(runMethods, (v) =>+ Urls.parseEnumParam(v, TRANSFORM_RUN_METHODS),+ ),+ sortColumn: Urls.parseEnumParam(+ sortColumn,+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,+ ),+ sortDirection: Urls.parseEnumParam(sortDirection, SORT_DIRECTIONS),+ };+}++export function getFilterOptions(+ params: Urls.TransformGraphRunListParams,+): TransformGraphRunFilterOptions {+ return {+ types: params.types,+ statuses: params.statuses,+ transformIds: params.transformIds,+ startTime: params.startTime,+ endTime: params.endTime,+ runMethods: params.runMethods,+ };+}++export function hasFilterOptions(+ options: TransformGraphRunFilterOptions,+): boolean {+ return (+ options.types != null ||+ options.statuses != null ||+ options.transformIds != null ||+ options.startTime != null ||+ options.endTime != null ||+ options.runMethods != null+ );+}++export function getSortOptions(+ params: Urls.TransformGraphRunListParams,+): TransformGraphRunSortOptions | undefined {+ return params.sortColumn != null && params.sortDirection != null+ ? { column: params.sortColumn, direction: params.sortDirection }+ : undefined;+}diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsxnew file mode 100644index 00000000000..c75b7e5beaa--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsx@@ -0,0 +1,76 @@+import { t } from "ttag";++import { skipToken, useListDagTransformsQuery } from "metabase/api";+import {+ Button,+ Center,+ Group,+ List,+ Loader,+ Modal,+ ScrollArea,+ Stack,+ Text,+} from "metabase/ui";+import type { TransformDagDirection, TransformId } from "metabase-types/api";++type RunDagConfirmModalProps = {+ transformId: TransformId;+ direction: TransformDagDirection | null;+ isConfirming?: boolean;+ onClose: () => void;+ onConfirm: () => void;+};++export function RunDagConfirmModal({+ transformId,+ direction,+ isConfirming,+ onClose,+ onConfirm,+}: RunDagConfirmModalProps) {+ const { data: transforms, isFetching } = useListDagTransformsQuery(+ direction ? { transformId, direction } : skipToken,+ );++ const title =+ direction === "upstream"+ ? t`Run this and all upstream transforms?`+ : t`Run this and all downstream transforms?`;++ return (+ <Modal+ opened={direction != null}+ title={title}+ padding="xl"+ onClose={onClose}+ >+ <Stack gap="lg">+ <Text>{t`These transforms will be run:`}</Text>+ {isFetching ? (+ <Center py="lg">+ <Loader />+ </Center>+ ) : (+ <ScrollArea.Autosize mah="50vh">+ <List>+ {(transforms ?? []).map((transform) => (+ <List.Item key={transform.id}>{transform.name}</List.Item>+ ))}+ </List>+ </ScrollArea.Autosize>+ )}+ <Group justify="flex-end">+ <Button onClick={onClose}>{t`Cancel`}</Button>+ <Button+ variant="filled"+ disabled={isFetching || isConfirming}+ onClick={onConfirm}+ >+ {t`Run all`}+ </Button>+ </Group>+ </Stack>+ </Modal>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsxindex cda55ac8cd6..48a8047251d 100644--- a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx@@ -1,9 +1,13 @@import { useDisclosure } from "@mantine/hooks";+import { useCallback, useEffect, useState } from "react";import { usePrevious } from "react-use";import { t } from "ttag";import {+ skipToken,useCancelCurrentTransformRunMutation,+ useListDagRunTransformRunsQuery,+ useRunTransformDagMutation,useRunTransformMutation,useUpdateTransformMutation,} from "metabase/api";@@ -13,13 +17,30 @@ import { useMetadataToasts } from "metabase/metadata/hooks";import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";import { useSelector } from "metabase/redux";import { Link } from "metabase/router";-import { Anchor, Box, Card, Divider, Group, Stack } from "metabase/ui";+import { POLLING_INTERVAL } from "metabase/transforms/constants";+import { isActiveRunStatus } from "metabase/transforms/utils";+import {+ Anchor,+ Box,+ Card,+ Divider,+ Group,+ Icon,+ Menu,+ Stack,+} from "metabase/ui";import * as Urls from "metabase/urls";import { isResourceNotFoundError } from "metabase/utils/errors";-import type { Transform, TransformTagId } from "metabase-types/api";+import type {+ Transform,+ TransformDagDirection,+ TransformDagRunId,+ TransformTagId,+} from "metabase-types/api";import {trackTransformRunTagsUpdated,+ trackTransformTriggerDagRun,trackTransformTriggerManualRun,} from "../../../analytics";import { RunButton } from "../../../components/RunButton";@@ -27,6 +48,7 @@ import { RunStatus } from "../../../components/RunStatus";import { TagMultiSelect } from "../../../components/TagMultiSelect";import { LogOutput } from "./LogOutput";+import { RunDagConfirmModal } from "./RunDagConfirmModal";type RunSectionProps = {transform: Transform;@@ -34,17 +56,65 @@ type RunSectionProps = {noTitle?: boolean;};+function useScheduledDagRun(transform: Transform) {+ const [scheduledDagRunId, setScheduledDagRunId] =+ useState<TransformDagRunId | null>(null);++ const { data: members, error } = useListDagRunTransformRunsQuery(+ scheduledDagRunId != null ? { dagRunId: scheduledDagRunId } : skipToken,+ {+ pollingInterval: scheduledDagRunId != null ? POLLING_INTERVAL : undefined,+ },+ );++ const dagFinished =+ members != null &&+ members.length > 0 &&+ !members.some((run) => isActiveRunStatus(run.status));++ const dagUnavailable = error != null;++ useEffect(() => {+ if (scheduledDagRunId != null && (dagFinished || dagUnavailable)) {+ setScheduledDagRunId(null);+ }+ }, [scheduledDagRunId, dagFinished, dagUnavailable]);++ const hasRunThisCycle = members?.some(+ (run) => run.transform_id === transform.id,+ );++ const schedule = useCallback(+ (dagRunId: TransformDagRunId) => setScheduledDagRunId(dagRunId),+ [],+ );++ return {+ isScheduled:+ scheduledDagRunId != null &&+ !dagFinished &&+ !dagUnavailable &&+ !hasRunThisCycle,+ schedule,+ };+}+export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {const isRemoteSyncReadOnly = useSelector(PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,);+ const { isScheduled, schedule } = useScheduledDagRun(transform);const content = (<><Stack><Group p="lg" justify="space-between">- <RunStatusSection transform={transform} />- <RunButtonSection transform={transform} readOnly={readOnly} />+ <RunStatusSection transform={transform} isScheduled={isScheduled} />+ <RunButtonSection+ transform={transform}+ readOnly={readOnly}+ onScheduled={schedule}+ /></Group><RunOutputSection transform={transform} /></Stack>@@ -82,9 +152,10 @@ export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {type RunStatusSectionProps = {transform: Transform;+ isScheduled: boolean;};-function RunStatusSection({ transform }: RunStatusSectionProps) {+function RunStatusSection({ transform, isScheduled }: RunStatusSectionProps) {const { id, last_run } = transform;const status = last_run?.status;@@ -97,6 +168,15 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {>{t`This run succeeded before it had a chance to cancel.`}</Box>);+ if (isScheduled) {+ return (+ <Group gap="sm" data-testid="run-status">+ <Icon c="text-secondary" name="clock" />+ <Box>{t`Scheduled to run as part of a reprocess run.`}</Box>+ </Group>+ );+ }+return (<Stack gap={0}><RunStatus@@ -107,6 +187,7 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {key="link"component={Link}to={Urls.transformRunList({ transformIds: [id] })}+ lh="inherit">{t`See all runs`}</Anchor>@@ -120,16 +201,25 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {type RunButtonSectionProps = {transform: Transform;readOnly?: boolean;+ onScheduled: (dagRunId: TransformDagRunId) => void;};-function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {+function RunButtonSection({+ transform,+ readOnly,+ onScheduled,+}: RunButtonSectionProps) {const [runTransform] = useRunTransformMutation();+ const [runTransformDag, { isLoading: isSubmittingDag }] =+ useRunTransformDagMutation();const [cancelTransform] = useCancelCurrentTransformRunMutation();- const { sendErrorToast } = useMetadataToasts();+ const { sendErrorToast, sendSuccessToast } = useMetadataToasts();const [isConfirmCancellationModalOpen,{ close: closeConfirmModal, open: openConfirmModal },] = useDisclosure(false);+ const [dagDirection, setDagDirection] =+ useState<TransformDagDirection | null>(null);const handleRun = async () => {trackTransformTriggerManualRun({ transformId: transform.id });@@ -139,6 +229,30 @@ function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {}};+ const handleRunDag = async () => {+ const direction = dagDirection;+ if (direction == null) {+ return;+ }+ trackTransformTriggerDagRun({ transformId: transform.id, direction });+ const { data, error } = await runTransformDag({+ id: transform.id,+ direction,+ });+ setDagDirection(null);+ if (error) {+ sendErrorToast(t`Failed to run transforms`);+ return;+ }+ if (data?.dag_run_id != null) {+ onScheduled(data.dag_run_id);+ } else {+ sendSuccessToast(+ t`A reprocess run for this transform is already in progress.`,+ );+ }+ };+const handleCancel = async () => {const { error } = await cancelTransform(transform.id);if (error && !isResourceNotFoundError(error)) {@@ -155,16 +269,37 @@ function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {onRun={handleRun}onCancel={openConfirmModal}isDisabled={readOnly}+ menuItems={+ readOnly ? undefined : (+ <>+ <Menu.Item onClick={() => setDagDirection("upstream")}>+ {t`Run this and all upstream transforms`}+ </Menu.Item>+ <Menu.Item onClick={() => setDagDirection("downstream")}>+ {t`Run this and all downstream transforms`}+ </Menu.Item>+ </>+ )+ }+ />+ <RunDagConfirmModal+ transformId={transform.id}+ direction={dagDirection}+ isConfirming={isSubmittingDag}+ onClose={() => setDagDirection(null)}+ onConfirm={handleRunDag}/><ConfirmModaltitle={t`Cancel this run?`}+ message={t`This requests the run to stop; it may take a moment to finish canceling.`}+ confirmButtonText={t`Cancel run`}+ closeButtonText={t`Keep running`}opened={isConfirmCancellationModalOpen}onClose={closeConfirmModal}onConfirm={() => {void handleCancel();closeConfirmModal();}}- closeButtonText={t`No`}/></>);diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsxnew file mode 100644index 00000000000..79a18ef9569--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsx@@ -0,0 +1,173 @@+import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";++import {+ setupListDagRunTransformRunsEndpoint,+ setupListDagTransformsEndpoint,+ setupListTransformTagsEndpoint,+ setupRunTransformDagEndpoint,+} from "__support__/server-mocks";+import { renderWithProviders, screen, waitFor } from "__support__/ui";+import { UndoListing } from "metabase/common/components/UndoListing";+import type {+ RunTransformDagResponse,+ Transform,+ TransformRunForJobRun,+} from "metabase-types/api";+import {+ createMockTransform,+ createMockTransformRun,+ createMockTransformRunForJobRun,+} from "metabase-types/api/mocks";++import { RunSection } from "./RunSection";++const TRANSFORM_ID = 10;+const DAG_RUN_ID = 1;++const DEFAULT_DAG_MEMBERS: TransformRunForJobRun[] = [+ createMockTransformRunForJobRun({+ id: 1,+ transform_id: 2,+ status: "started",+ }),+];++function setup({+ transform,+ runDagResponse,+ dagMembers = DEFAULT_DAG_MEMBERS,+}: {+ transform?: Transform;+ runDagResponse?: RunTransformDagResponse;+ dagMembers?: TransformRunForJobRun[];+} = {}) {+ const testTransform =+ transform ?? createMockTransform({ id: TRANSFORM_ID, last_run: null });++ setupListTransformTagsEndpoint([]);+ setupListDagTransformsEndpoint(TRANSFORM_ID, [+ { id: 1, name: "Upstream A" },+ { id: 2, name: "Upstream B" },+ { id: TRANSFORM_ID, name: testTransform.name },+ ]);+ setupRunTransformDagEndpoint(TRANSFORM_ID, runDagResponse);+ setupListDagRunTransformRunsEndpoint(DAG_RUN_ID, dagMembers);++ renderWithProviders(+ <>+ <RunSection transform={testTransform} />+ <UndoListing />+ </>,+ );+}++async function triggerUpstreamRun() {+ await userEvent.click(screen.getByTestId("run-options-button"));+ await userEvent.click(+ await screen.findByText("Run this and all upstream transforms"),+ );+ await screen.findByText("Upstream A");+ await userEvent.click(screen.getByRole("button", { name: "Run all" }));+}++describe("RunSection DAG run flow", () => {+ it("offers upstream and downstream run options", async () => {+ setup();+ await userEvent.click(screen.getByTestId("run-options-button"));++ expect(+ await screen.findByText("Run this and all upstream transforms"),+ ).toBeInTheDocument();+ expect(+ screen.getByText("Run this and all downstream transforms"),+ ).toBeInTheDocument();+ });++ it("previews the transforms and triggers a run-dag with the chosen direction", async () => {+ setup();+ await userEvent.click(screen.getByTestId("run-options-button"));+ await userEvent.click(+ await screen.findByText("Run this and all upstream transforms"),+ );++ expect(+ await screen.findByText("Run this and all upstream transforms?"),+ ).toBeInTheDocument();+ expect(await screen.findByText("Upstream A")).toBeInTheDocument();+ expect(screen.getByText("Upstream B")).toBeInTheDocument();++ await userEvent.click(screen.getByRole("button", { name: "Run all" }));++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called(+ `path:/api/transform/${TRANSFORM_ID}/run-dag`,+ ),+ ).toBe(true);+ });+ const call = fetchMock.callHistory.lastCall(+ `path:/api/transform/${TRANSFORM_ID}/run-dag`,+ );+ const rawBody = call?.options?.body;+ const body = JSON.parse(typeof rawBody === "string" ? rawBody : "{}");+ expect(body.direction).toBe("upstream");+ });++ it("shows a scheduled state after triggering a DAG run", async () => {+ setup();+ await triggerUpstreamRun();++ expect(+ await screen.findByText("Scheduled to run as part of a reprocess run."),+ ).toBeInTheDocument();+ });++ it("stops showing the scheduled state once this transform's own run has completed mid-DAG", async () => {+ setup({+ transform: createMockTransform({+ id: TRANSFORM_ID,+ last_run: createMockTransformRun({+ status: "succeeded",+ end_time: null,+ }),+ }),+ dagMembers: [+ createMockTransformRunForJobRun({+ id: 1,+ transform_id: 2,+ status: "started",+ }),+ createMockTransformRunForJobRun({+ id: 2,+ transform_id: TRANSFORM_ID,+ status: "succeeded",+ }),+ ],+ });+ await triggerUpstreamRun();++ expect(+ await screen.findByText("Last ran successfully."),+ ).toBeInTheDocument();+ expect(+ screen.queryByText("Scheduled to run as part of a reprocess run."),+ ).not.toBeInTheDocument();+ });++ it("notifies the user and skips the scheduled state when nothing was run", async () => {+ setup({+ runDagResponse: { message: "DAG run started", dag_run_id: null },+ });+ await triggerUpstreamRun();++ expect(+ await screen.findByText(+ "A reprocess run for this transform is already in progress.",+ ),+ ).toBeInTheDocument();+ expect(+ screen.queryByText("Scheduled to run as part of a reprocess run."),+ ).not.toBeInTheDocument();+ });+});diff --git a/frontend/src/metabase/transforms/routes.tsx b/frontend/src/metabase/transforms/routes.tsxindex bbef73f2d40..4bfd4e7a16b 100644--- a/frontend/src/metabase/transforms/routes.tsx+++ b/frontend/src/metabase/transforms/routes.tsx@@ -16,7 +16,9 @@ import {NewQueryTransformPage,} from "./pages/NewTransformPage";import { RunListPage } from "./pages/RunListPage";+import { RunsPage } from "./pages/RunsPage";import { TransformDependenciesPage } from "./pages/TransformDependenciesPage";+import { TransformGraphRunListPage } from "./pages/TransformGraphRunListPage";import { TransformIndexesPage } from "./pages/TransformIndexesPage";import { TransformListPage } from "./pages/TransformListPage";import { TransformQueryPage } from "./pages/TransformQueryPage";@@ -26,6 +28,9 @@ import { TransformsNotDisabled } from "./route-guards";const RoutedTransformListPage = withRouteProps(TransformListPage);const RoutedRunListPage = withRouteProps(RunListPage);+const RoutedTransformGraphRunListPage = withRouteProps(+ TransformGraphRunListPage,+);const RoutedNewJobPage = withRouteProps(NewJobPage);const RoutedJobPage = withRouteProps(JobPage);const RoutedJobRunListPage = withRouteProps(JobRunListPage);@@ -44,7 +49,10 @@ export function getDataStudioTransformRoutes() {return (<Route element={<TransformsNotDisabled />}><Route index element={<RoutedTransformListPage />} />- <Route path="runs" element={<RoutedRunListPage />} />+ <Route path="runs" element={<RunsPage />}>+ <Route index element={<RoutedTransformGraphRunListPage />} />+ <Route path="individual" element={<RoutedRunListPage />} />+ </Route><Route path="jobs" element={<JobSectionLayout />}><Route index element={<JobListPage />} /><Route path="new" element={<RoutedNewJobPage />} />diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.tsindex 2a284aeb6ed..5c312ce795a 100644--- a/frontend/src/metabase/transforms/utils.ts+++ b/frontend/src/metabase/transforms/utils.ts@@ -122,6 +122,12 @@ export function isErrorStatus(status: TransformRunStatus | null) {return status === "failed" || status === "timeout";}+export function isActiveRunStatus(+ status: TransformRunStatus | null | undefined,+) {+ return status === "started" || status === "canceling";+}+export function isTransformRunning(transform: Transform) {const lastRun = transform.last_run;return lastRun?.status === "started";diff --git a/frontend/src/metabase/urls/transforms.ts b/frontend/src/metabase/urls/transforms.tsindex a705c3afb0b..89d9ba62cbb 100644--- a/frontend/src/metabase/urls/transforms.ts+++ b/frontend/src/metabase/urls/transforms.ts@@ -1,9 +1,13 @@+import _ from "underscore";+import type {CardId,CollectionId,DatabaseId,SortDirection,TableId,+ TransformGraphRunSortColumn,+ TransformGraphRunType,TransformId,TransformJobId,TransformJobRunSortColumn,@@ -17,6 +21,7 @@ import type {const TRANSFORMS_ROOT_URL = `/data-studio/transforms`;const JOBS_ROOT_URL = `${TRANSFORMS_ROOT_URL}/jobs`;const RUNS_ROOT_URL = `${TRANSFORMS_ROOT_URL}/runs`;+const INDIVIDUAL_RUNS_ROOT_URL = `${RUNS_ROOT_URL}/individual`;const LIBRARY_ROOT_URL = `${TRANSFORMS_ROOT_URL}/library`;export type TransformPythonLibraryParams = {@@ -198,12 +203,91 @@ export function transformRunList({const queryString = searchParams.toString();if (queryString.length > 0) {- return `${RUNS_ROOT_URL}?${queryString}`;+ return `${INDIVIDUAL_RUNS_ROOT_URL}?${queryString}`;} else {- return RUNS_ROOT_URL;+ return INDIVIDUAL_RUNS_ROOT_URL;}}+export type TransformGraphRunListParams = {+ page?: number;+ types?: TransformGraphRunType[];+ statuses?: TransformRunStatus[];+ transformIds?: TransformId[];+ startTime?: string;+ endTime?: string;+ runMethods?: TransformRunMethod[];+ sortColumn?: TransformGraphRunSortColumn;+ sortDirection?: SortDirection;+};++export function transformGraphRunList({+ page,+ types,+ statuses,+ transformIds,+ startTime,+ endTime,+ runMethods,+ sortColumn,+ sortDirection,+}: TransformGraphRunListParams = {}) {+ const searchParams = new URLSearchParams();+ if (page != null) {+ searchParams.set("page", String(page));+ }+ types?.forEach((type) => {+ searchParams.append("types", type);+ });+ statuses?.forEach((status) => {+ searchParams.append("statuses", status);+ });+ transformIds?.forEach((transformId) => {+ searchParams.append("transform-ids", String(transformId));+ });+ if (startTime != null) {+ searchParams.set("start-time", startTime);+ }+ if (endTime != null) {+ searchParams.set("end-time", endTime);+ }+ runMethods?.forEach((runMethod) => {+ searchParams.append("run-methods", runMethod);+ });+ if (sortColumn != null) {+ searchParams.set("sort-column", sortColumn);+ }+ if (sortDirection != null) {+ searchParams.set("sort-direction", sortDirection);+ }++ const queryString = searchParams.toString();+ return queryString.length > 0+ ? `${RUNS_ROOT_URL}?${queryString}`+ : RUNS_ROOT_URL;+}++export type CommonRunListParams = {+ statuses?: TransformRunStatus[];+ transformIds?: TransformId[];+ startTime?: string;+ endTime?: string;+ runMethods?: TransformRunMethod[];+};++export function pickCommonRunListParams(+ params: CommonRunListParams,+): CommonRunListParams {+ return _.pick(+ params,+ "statuses",+ "transformIds",+ "startTime",+ "endTime",+ "runMethods",+ );+}+export function transformPythonLibrary({ path }: TransformPythonLibraryParams) {return `${LIBRARY_ROOT_URL}/${path}`;}diff --git a/frontend/test/__support__/server-mocks/transform.ts b/frontend/test/__support__/server-mocks/transform.tsindex b66c6891a15..3563d4e44f1 100644--- a/frontend/test/__support__/server-mocks/transform.ts+++ b/frontend/test/__support__/server-mocks/transform.ts@@ -1,11 +1,15 @@import fetchMock from "fetch-mock";import type {+ DagTransform,Dataset,InspectorLensId,+ ListTransformGraphRunsResponse,ListTransformJobRunsResponse,ListTransformRunsResponse,+ RunTransformDagResponse,Transform,+ TransformDagRunId,TransformId,TransformJob,TransformJobId,@@ -20,6 +24,12 @@ export function setupListTransformRunsEndpoint(fetchMock.get(`path:/api/transform/run`, response);}+export function setupListTransformGraphRunsEndpoint(+ response: ListTransformGraphRunsResponse,+) {+ fetchMock.get(`path:/api/transform/runs`, response);+}+export function setupListTransformsEndpoint(transforms: Transform[]) {fetchMock.get(`path:/api/transform`, transforms);}@@ -84,6 +94,40 @@ export function setupListJobRunTransformRunsEndpoint();}+export function setupListDagTransformsEndpoint(+ transformId: TransformId,+ transforms: DagTransform[],+) {+ fetchMock.get(+ `path:/api/transform/${transformId}/dag-transforms`,+ transforms,+ );+}++export function setupRunTransformDagEndpoint(+ transformId: TransformId,+ response: RunTransformDagResponse = {+ message: "DAG run started",+ dag_run_id: 1,+ },+) {+ fetchMock.post(`path:/api/transform/${transformId}/run-dag`, response);+}++export function setupListDagRunTransformRunsEndpoint(+ dagRunId: TransformDagRunId,+ runs: TransformRunForJobRun[] | (() => TransformRunForJobRun[]),+) {+ fetchMock.get(`path:/api/transform-dag-run/${dagRunId}/transform-runs`, runs);+}++export function setupCancelJobRunEndpoint(+ jobId: TransformJobId,+ runId: TransformJobRunId,+) {+ fetchMock.post(`path:/api/transform-job/${jobId}/runs/${runId}/cancel`, 204);+}+export function setupCreateTransformJobEndpoint(job: TransformJob) {fetchMock.post("path:/api/transform-job", job);}diff --git a/resources/migrations/064/20260703_transform_dag_run.yaml b/resources/migrations/064/20260703_transform_dag_run.yamlnew file mode 100644index 00000000000..6393c205d02--- /dev/null+++ b/resources/migrations/064/20260703_transform_dag_run.yaml@@ -0,0 +1,349 @@+## DAG-reprocess runs: reprocessing the transitive dependency DAG rooted at a single transform+## (upstream or downstream). These are coordinated multi-transform runs, like scheduled job runs,+## but triggered manually from one seed transform rather than by a job. They get their own table+## (transform_dag_run) rather than overloading transform_job_run, and each member transform_run is+## linked back to its coordinating DAG run via transform_run.dag_run_id (mirroring job_run_id).++databaseChangeLog:++ - changeSet:+ id: v64.oxRKX6+ author: bronsa+ comment: Create transform_dag_run table to track manual DAG-reprocess runs seeded from a single transform+ preConditions:+ - onFail: MARK_RAN+ - not:+ - tableExists:+ tableName: transform_dag_run+ changes:+ - createTable:+ tableName: transform_dag_run+ remarks: Tracks manual DAG-reprocess runs (reprocessing a transform's upstream/downstream dependency DAG)+ columns:+ - column:+ name: id+ type: int+ remarks: Unique ID+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_transform_id+ type: int+ remarks: The seed transform the DAG reprocess was triggered from (null if it was later deleted)+ constraints:+ nullable: true+ - column:+ name: source_transform_name+ type: ${text.type}+ remarks: Name of the seed transform at run start, preserved after its deletion+ constraints:+ nullable: true+ - column:+ name: source_transform_entity_id+ type: char(21)+ remarks: entity_id of the seed transform at run start, for potential reconnection when it is recreated+ constraints:+ nullable: true+ - column:+ name: direction+ type: varchar(20)+ remarks: DAG traversal direction (upstream/downstream) that selected the transforms to run+ constraints:+ nullable: false+ - column:+ name: transform_count+ type: int+ remarks: Number of transforms the run's plan selected to run, for display in run listings+ constraints:+ nullable: true+ - column:+ name: status+ type: varchar(32)+ remarks: Current status of the run (started, succeeded, failed, timeout, canceled)+ constraints:+ nullable: false+ - column:+ name: is_active+ type: ${boolean.type}+ remarks: True only for currently running DAG runs, null for the others+ constraints:+ nullable: true+ defaultValue: true+ - column:+ name: start_time+ type: ${timestamp_type}+ remarks: When the DAG run started+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ - column:+ name: end_time+ type: ${timestamp_type}+ remarks: When the DAG run completed (null if still running)+ constraints:+ nullable: true+ - column:+ name: message+ type: ${text.type}+ remarks: Human-readable message about the run; may contain the error message when the run failed+ constraints:+ nullable: true+ - column:+ name: user_id+ type: int+ remarks: The user who triggered the DAG run (null if the user was later deleted)+ constraints:+ nullable: true+ - column:+ name: last_heartbeat+ type: ${timestamp_type}+ defaultValueComputed: current_timestamp+ remarks: Heartbeat timestamp stamped periodically while the run executes; a stale value means the owning process died+ constraints:+ nullable: false+ - column:+ name: created_at+ type: ${timestamp_type}+ remarks: The timestamp this DAG run was created.+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ - column:+ name: updated_at+ type: ${timestamp_type}+ remarks: The timestamp this DAG run was last updated.+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ rollback:+ - dropTable:+ tableName: transform_dag_run++ - changeSet:+ id: v64.btfmlX+ author: bronsa+ comment: Index transform_dag_run on source_transform_id for per-transform DAG run history lookups (also backs the FK below)+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_source_transform_id+ changes:+ - createIndex:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_source_transform_id+ columns:+ - column:+ name: source_transform_id+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.dQGhvK+ author: bronsa+ comment: >-+ FK from transform_dag_run.source_transform_id to transform; SET NULL on delete so DAG-run+ history survives the transform (as for transform_run.transform_id), displayable via the+ run-start name snapshot+ preConditions:+ - onFail: MARK_RAN+ - not:+ - foreignKeyConstraintExists:+ foreignKeyTableName: transform_dag_run+ foreignKeyName: fk_transform_dag_run_source_transform_id+ changes:+ - addForeignKeyConstraint:+ constraintName: fk_transform_dag_run_source_transform_id+ baseTableName: transform_dag_run+ baseColumnNames: source_transform_id+ referencedTableName: transform+ referencedColumnNames: id+ onDelete: SET NULL+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.yY6UC9+ author: bronsa+ comment: Index transform_dag_run on user_id (backs the FK below and covers per-user run history lookups)+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_user_id+ changes:+ - createIndex:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_user_id+ columns:+ - column:+ name: user_id+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.BTINKo+ author: bronsa+ comment: FK from transform_dag_run.user_id to core_user; null it out when the user is deleted so run history survives+ preConditions:+ - onFail: MARK_RAN+ - not:+ - foreignKeyConstraintExists:+ foreignKeyTableName: transform_dag_run+ foreignKeyName: fk_transform_dag_run_user_id+ changes:+ - addForeignKeyConstraint:+ constraintName: fk_transform_dag_run_user_id+ baseTableName: transform_dag_run+ baseColumnNames: user_id+ referencedTableName: core_user+ referencedColumnNames: id+ onDelete: SET NULL+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.ohN51V+ author: bronsa+ comment: Index transform_dag_run on (is_active, last_heartbeat) so the every-minute reaper scan is O(active rows)+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_active_runs+ changes:+ - createIndex:+ tableName: transform_dag_run+ indexName: idx_transform_dag_run_active_runs+ columns:+ - column:+ name: is_active+ - column:+ name: last_heartbeat+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.wQe4Fn+ author: bronsa+ comment: >-+ Unique constraint so at most one DAG run can be active per seed transform.+ changes:+ - addUniqueConstraint:+ tableName: transform_dag_run+ constraintName: idx_unique_active_transform_dag_run+ columnNames: source_transform_id, is_active+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table++ - changeSet:+ id: v64.HUpfDD+ author: bronsa+ comment: >-+ Add dag_run_id to transform_run to link each transform execution to its coordinating DAG run+ (mirrors job_run_id, which links to transform_job_run). Exactly one of the two is set for a+ coordinated member run; both are null for a standalone single-transform run.+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_run+ columnName: dag_run_id+ changes:+ - addColumn:+ tableName: transform_run+ columns:+ - column:+ name: dag_run_id+ type: bigint+ remarks: FK to transform_dag_run; links each transform execution to its coordinating DAG run+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: transform_run+ columnName: dag_run_id++ - changeSet:+ id: v64.tOY28l+ author: bronsa+ comment: Index transform_run on dag_run_id for efficient drill-down of a DAG run's member transform runs+ preConditions:+ - onFail: MARK_RAN+ - not:+ - indexExists:+ tableName: transform_run+ indexName: idx_transform_run_dag_run_id+ changes:+ - createIndex:+ tableName: transform_run+ indexName: idx_transform_run_dag_run_id+ columns:+ - column:+ name: dag_run_id+ rollback:+ - dropIndex:+ tableName: transform_run+ indexName: idx_transform_run_dag_run_id++ - changeSet:+ id: v64.cH9mZs+ author: bronsa+ comment: >-+ Snapshot the job's name on transform_job_run at run start, as transform_run does for its+ transform. transform_job_run.job_id has no FK, so job runs already outlive their job — but+ namelessly; the snapshot keeps them displayable in run history after the job is deleted.+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_job_run+ columnName: job_name+ changes:+ - addColumn:+ tableName: transform_job_run+ columns:+ - column:+ name: job_name+ type: ${text.type}+ remarks: Name of the job at run start, preserved after its deletion+ rollback:+ - dropColumn:+ tableName: transform_job_run+ columnName: job_name++ - changeSet:+ id: v64.gW3nQd+ author: bronsa+ comment: Snapshot the job's entity_id on transform_job_run at run start, for potential reconnection when a deleted job is recreated+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_job_run+ columnName: job_entity_id+ changes:+ - addColumn:+ tableName: transform_job_run+ columns:+ - column:+ name: job_entity_id+ type: char(21)+ remarks: entity_id of the job at run start, for potential reconnection when it is recreated+ rollback:+ - dropColumn:+ tableName: transform_job_run+ columnName: job_entity_id++ - changeSet:+ id: v64.aY5kXm+ author: bronsa+ comment: Backfill the job name/entity_id snapshots on existing transform_job_run rows from their live jobs (this table predates the branch)+ changes:+ - sql:+ sql: >-+ UPDATE transform_job_run+ SET job_name = (SELECT name FROM transform_job WHERE transform_job.id = transform_job_run.job_id),+ job_entity_id = (SELECT entity_id FROM transform_job WHERE transform_job.id = transform_job_run.job_id)+ rollback:+ - sql:+ sql: UPDATE transform_job_run SET job_name = NULL, job_entity_id = NULLdiff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex 3c80453c6f5..bf474669e94 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -243,6 +243,7 @@"/timeline" (+auth metabase.timeline.api/timeline-routes)"/timeline-event" (+auth metabase.timeline.api/timeline-event-routes)"/transform" (+auth metabase.transforms-rest.api.transform/routes)+ "/transform-dag-run" (+auth metabase.transforms-rest.api.transform/transform-dag-run-routes)"/transform-job" (+auth metabase.transforms-rest.api.transform/transform-job-routes)"/transform-tag" (+auth metabase.transforms-rest.api.transform/transform-tag-routes)"/typed-schemas" (+auth metabase.typed-schemas.api/routes)diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 439f80e30d8..21cf56883ae 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -130,6 +130,7 @@:model/Timeline metabase.timeline.models.timeline:model/TimelineEvent metabase.timeline.models.timeline-event:model/Transform metabase.transforms.models.transform+ :model/TransformDagRun metabase.transforms.models.dag-run:model/TransformJob metabase.transforms.models.transform-job:model/TransformJobRun metabase.transforms.models.job-run:model/TransformJobTransformTag metabase.transforms.models.transform-job-transform-tagdiff --git a/src/metabase/transforms/coordinated_run.clj b/src/metabase/transforms/coordinated_run.cljnew file mode 100644index 00000000000..258911f6b66--- /dev/null+++ b/src/metabase/transforms/coordinated_run.clj@@ -0,0 +1,99 @@+(ns metabase.transforms.coordinated-run+ "Shared lifecycle operations for the two coordinated-run models, `transform_job_run` and+ `transform_dag_run`. Both track a multi-transform run with the same status/heartbeat lifecycle and+ differ only in their extra columns and member-run FK; helpers here are parameterized by the Toucan+ `model`."+ (:require+ [metabase.run-tracking.core :as rt]+ [metabase.transforms.canceling :as canceling]+ [metabase.transforms.models.transform-run-cancelation :as transform-run-cancelation]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn add-run-activity!+ "Note that a run has had activity (touches `updated_at`)."+ [model run-id]+ (t2/update! model :id run-id :is_active true {:updated_at :%now}))++(defn succeed-started-run!+ "Mark a started run as successfully completed."+ ([model run-id]+ (succeed-started-run! model run-id {}))+ ([model run-id properties]+ (t2/update! model+ :id run-id+ :is_active true+ (merge {:end_time :%now}+ properties+ {:status :succeeded+ :is_active nil}))))++(defn fail-started-run!+ "Mark the started active run as failed and inactive."+ [model run-id properties]+ (t2/update! model+ :id run-id+ :is_active true+ (merge {:end_time :%now}+ properties+ {:status :failed+ :is_active nil})))++(defn cancel-started-run!+ "Mark an active run as canceled; a finished run is never resurrected into a canceled state.+ Returns the number of rows updated — 0 if the run had already finished."+ [model run-id]+ (t2/update! model+ :id run-id+ :is_active true+ {:status :canceled+ :is_active nil+ :end_time :%now+ :message "Canceled"}))++(defn cancel!+ "Cancel an in-progress coordinated run: mark it canceled and request cancellation of its+ still-active member transform runs — via the in-process cancel channel, plus a cancelation row so+ the cancel-runs task reaches members running on other nodes. Returns true if the run was active+ and is now canceled, false if it had already finished."+ [model member-fk run-id]+ (boolean+ (when (pos? (cancel-started-run! model run-id))+ (doseq [member-run-id (t2/select-pks-vec :model/TransformRun member-fk run-id :is_active true)]+ (transform-run-cancelation/mark-cancel-started-run! member-run-id)+ (canceling/chan-signal-cancel! member-run-id))+ true)))++(defn heartbeat-runs!+ "Stamp `last_heartbeat = now` on the given still-active run-ids."+ [model run-ids]+ (rt/heartbeat-ids! model [:= :is_active true] :last_heartbeat run-ids))++(defn reap-orphaned-runs!+ "Time out active runs whose `last_heartbeat` is older than `stale-minutes` (their coordinator+ process is presumed dead). `type-tag` (\"job\"/\"dag\") tags the emitted metrics. Returns the rows+ that were timed out so callers can notify."+ [model type-tag stale-minutes]+ (rt/reap-orphaned!+ {:model model+ :active [:= :is_active true]+ :stale [:< :last_heartbeat (rt/cutoff stale-minutes :minute)]+ :terminal {:status "timeout" :end_time :%now :is_active nil :message "Timed out: crashed"}+ :metrics {:total-metric :metabase-transforms/timeouts-total+ :latency-metric :metabase-transforms/timeout-detection-latency-ms+ :tags {:type type-tag}+ :latency-column :last_heartbeat+ :timeout-ms (rt/unit->ms stale-minutes :minute)}}))++(defn heartbeat-and-reconcile!+ "Stamp a heartbeat on every active run whose id is a key of `active-runs-atom` (the runs this+ process coordinates), then deliver the `gone` promise of any that were terminated externally so+ their coordinator aborts."+ [model active-runs-atom]+ (rt/heartbeat-and-reconcile! {:model model+ :active [:= :is_active true]+ :ids (keys @active-runs-atom)+ :heartbeat! #(heartbeat-runs! model %)+ :on-gone (fn [run-id]+ (some-> (get @active-runs-atom run-id) (deliver true)))}))diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex b1db52505e2..cc453bfbffa 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -5,8 +5,10 @@[metabase.transforms-base.util][metabase.transforms.canceling][metabase.transforms.crud]+ [metabase.transforms.dag][metabase.transforms.execute][metabase.transforms.jobs]+ [metabase.transforms.models.dag-run][metabase.transforms.models.job-run][metabase.transforms.models.transform][metabase.transforms.models.transform-job]@@ -14,6 +16,7 @@[metabase.transforms.models.transform-run-cancelation][metabase.transforms.models.transform-tag][metabase.transforms.notification]+ [metabase.transforms.run-listing][metabase.transforms.schedule][metabase.transforms.settings][metabase.transforms.usage]@@ -57,7 +60,12 @@get-transform-cycle][metabase.transforms.jobsrun-job!+ cancel-job-run!job-transforms]+ [metabase.transforms.dag+ run-dag!+ cancel-dag-run!+ dag-run-transforms][metabase.transforms.schedulevalidate-cron-expressioninitialize-job!@@ -74,6 +82,11 @@[metabase.transforms.models.job-runpaged-job-runstransform-runs-for-job-run]+ [metabase.transforms.models.dag-run+ transform-runs-for-dag-run]+ [metabase.transforms.run-listing+ paged-run-summaries+ present-run-summaries][metabase.transforms.models.transform-run-cancelationmark-cancel-started-run!][metabase.transforms.models.transform-jobdiff --git a/src/metabase/transforms/dag.clj b/src/metabase/transforms/dag.cljnew file mode 100644index 00000000000..de3d6ae6b5e--- /dev/null+++ b/src/metabase/transforms/dag.clj@@ -0,0 +1,184 @@+(ns metabase.transforms.dag+ "Manual DAG-reprocess runs: reprocessing the transitive dependency DAG rooted at a single seed+ transform, upstream or downstream. This reuses the generic multi-transform coordinator in+ [[metabase.transforms.jobs]] (which honors the DAG, lanes, cancelation and timeouts) but tracks+ progress in its own `transform_dag_run` table rather than `transform_job_run`, and links member+ transform runs via `transform_run.dag_run_id`."+ (:require+ [clojure.set :as set]+ [metabase.run-tracking.core :as rt]+ [metabase.task.core :as task]+ [metabase.tracing.core :as tracing]+ [metabase.transforms-base.ordering :as ordering]+ [metabase.transforms.coordinated-run :as coordinated-run]+ [metabase.transforms.jobs :as jobs]+ [metabase.transforms.models.dag-run :as dag-run]+ [metabase.transforms.util :as transforms.u]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private dag-heartbeat-stale-minutes+ "A DAG run whose coordinator hasn't heartbeat in this many minutes is presumed dead and reaped."+ 5)++(defonce ^:private dag-active-runs+ ;; dag-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped) so+ ;; its coordinator aborts. Separate from the job-run atom in [[metabase.transforms.jobs]] because+ ;; the two id spaces (transform_dag_run vs transform_job_run) are independent.+ (atom {}))++;;; ------------------------------------------- Planning -------------------------------------------++(defn- dependents-graph+ "Reverse a forward dependency map (id -> #{deps it reads}) into a dependents map+ (id -> #{transforms that read it})."+ [forward-deps]+ (reduce-kv (fn [acc id deps]+ (reduce #(update %1 %2 (fnil conj #{}) id) acc deps))+ {} forward-deps))++(defn- reachable+ "Set of nodes reachable from `start` in `graph` (id -> #{neighbor-ids}), including `start`."+ [graph start]+ (loop [queue [start] seen #{}]+ (if-let [node (first queue)]+ (if (seen node)+ (recur (rest queue) seen)+ (recur (into (rest queue) (get graph node)) (conj seen node)))+ seen)))++(defn- lean-transforms+ "Transforms with only the columns the ordering walk needs — avoids loading every transform's full+ row just to compute the dependency graph (full rows are fetched only for the resulting closure)."+ []+ (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies]))++(defn- full-transforms [ids]+ (if (seq ids)+ (t2/select :model/Transform :id [:in ids])+ []))++(defn- dag-run-plan+ "Compute both the set of transform-ids a DAG reprocess should run and the execution plan to run+ them. Returns `{:transform-ids #{...} :plan {:order [...] :deps {...}}}`.++ `:upstream` — seed + all transforms the seed transitively depends on. This is exactly the closure+ `transform-ordering` walks from the seed, so the walk's `:dependencies` map is both+ the id set (its keys) and the plan input. The whole closure runs.+ `:downstream` — seed + all transforms that transitively depend on the seed. The ordering code only+ walks toward dependencies, so we build the full forward graph once and reverse it to+ find dependents. Only the downstream set runs — the dependents' *other* upstream+ inputs are already-materialized tables and must NOT be reprocessed, so we keep only+ the dependency edges that stay within the downstream set. Dropping the external edges+ preserves ordering (the seed still runs before anything that depends on it) while+ leaving those inputs untouched."+ [seed-id direction]+ (let [lean (lean-transforms)]+ (case direction+ :upstream+ (let [{:keys [dependencies uncached]} (ordering/transform-ordering #{seed-id} lean)+ ids (set (keys dependencies))]+ (ordering/persist-table-dependencies! uncached)+ {:transform-ids ids+ :plan (jobs/dependencies->plan dependencies (full-transforms ids))})++ :downstream+ (let [{:keys [dependencies uncached]} (ordering/transform-ordering (into #{} (map :id) lean) lean)+ _ (ordering/persist-table-dependencies! uncached)+ downstream-ids (reachable (dependents-graph dependencies) seed-id)+ deps (into {}+ (map (fn [id]+ [id (set/intersection (get dependencies id #{}) downstream-ids)]))+ downstream-ids)]+ {:transform-ids downstream-ids+ :plan (jobs/dependencies->plan deps (full-transforms downstream-ids))}))))++(defn dag-run-transforms+ "Return the transforms a DAG reprocess from `transform-id` would run, in planned execution order.+ `direction` is as in [[run-dag!]]."+ [transform-id direction]+ (-> (dag-run-plan transform-id direction) :plan :order vec))++;;; ------------------------------------------- Execution -------------------------------------------++(defn- start-dag-run-unless-racing!+ "Insert the DAG run row, or return nil if a concurrent trigger for the same seed transform won+ the `(source_transform_id, is_active)` uniqueness race."+ [transform-id direction user-id transform-count]+ (try+ (dag-run/start-dag-run! transform-id direction user-id transform-count)+ (catch Exception e+ (when-not (transforms.u/duplicate-key-violation? e)+ (throw e)))))++(defn run-dag!+ "Run all transforms in the dependency DAG rooted at `transform-id`, tracking progress in a+ `transform_dag_run` row. Blocks until the run finishes; runs are always manual.++ `direction` selects which transforms are included:+ - `:upstream` — the seed plus all transforms it transitively depends on+ - `:downstream` — the seed plus all transforms that transitively depend on it++ `:start-promise`, if provided, is delivered `[:started dag-run-id]` once the run row is created,+ `nil` if nothing was run (already running / empty closure), or a Throwable on a pre-start failure.+ Returns the dag-run-id, or nil if nothing was executed."+ [transform-id {:keys [direction user-id start-promise]}]+ (try+ (if (dag-run/running-run-for-source-transform-id transform-id)+ (do (log/info "Not executing DAG run for transform" (pr-str transform-id) "because one is already running")+ (some-> start-promise (deliver nil))+ nil)+ (let [{:keys [transform-ids plan]} (dag-run-plan transform-id direction)]+ (if (empty? transform-ids)+ (do (log/info "Skipping DAG run for transform" (pr-str transform-id) "because no transforms found in closure")+ (some-> start-promise (deliver nil))+ nil)+ (if-let [{run-id :id} (start-dag-run-unless-racing! transform-id direction user-id+ (count transform-ids))]+ (do (some-> start-promise (deliver [:started run-id]))+ (tracing/with-span :tasks "task.transform.run-dag" {:transform/id transform-id+ :transform/direction (name direction)+ :transform/count (count transform-ids)}+ (jobs/execute-coordinated-run! :model/TransformDagRun run-id transform-ids plan+ (format "DAG run for transform %s" (pr-str transform-id))+ {:run-method :manual+ :user-id user-id+ :parent-run-type :dag+ :active-runs-atom dag-active-runs}))+ run-id)+ (do (log/info "Not executing DAG run for transform" (pr-str transform-id) "because one is already running")+ (some-> start-promise (deliver nil))+ nil)))))+ (catch Throwable t+ ;; unblock any caller waiting on the promise on a pre-start failure+ (some-> start-promise (deliver t))+ (throw t))))++(defn cancel-dag-run!+ "Cancel an in-progress DAG run and request cancellation of its still-running member transform runs.+ Returns true if the run was active and is now canceled, false if it had already finished."+ [run-id]+ (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id))++;;; ------------------------------------------- Heartbeat / reaping -------------------------------------------++(defn- heartbeat-and-reconcile-runs!+ "Stamp a heartbeat on every active DAG run this process is coordinating, then deliver the `gone`+ promise of any that were terminated externally so their coordinator aborts."+ []+ (coordinated-run/heartbeat-and-reconcile! :model/TransformDagRun dag-active-runs))++(defn- reap-orphaned-runs!+ "Reap DAG runs whose coordinator process died (stale heartbeat)."+ []+ (coordinated-run/reap-orphaned-runs! :model/TransformDagRun "dag" dag-heartbeat-stale-minutes))++(defmethod task/init! ::TransformDagRunHeartbeat [_]+ (rt/start-heartbeat! heartbeat-and-reconcile-runs! 1))++(defmethod task/init! ::TransformDagRunReaper [_]+ (rt/schedule-reaper! {:job-key "metabase.transforms.dag.reaper-job"+ :label "transform DAG run"+ :reap-fn #'reap-orphaned-runs!}))diff --git a/src/metabase/transforms/init.clj b/src/metabase/transforms/init.cljindex 3678691726e..484206d4a41 100644--- a/src/metabase/transforms/init.clj+++ b/src/metabase/transforms/init.clj@@ -1,7 +1,9 @@(ns metabase.transforms.init(:require[metabase.transforms.canceling]+ [metabase.transforms.dag][metabase.transforms.jobs]+ [metabase.transforms.models.dag-run][metabase.transforms.models.job-run][metabase.transforms.models.transform][metabase.transforms.models.transform-job]diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex 7125cabfb46..135f13acdb6 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -12,6 +12,7 @@[metabase.transforms-base.ordering :as transforms-base.ordering][metabase.transforms-base.util :as transforms-base.u][metabase.transforms.canceling :as canceling]+ [metabase.transforms.coordinated-run :as coordinated-run][metabase.transforms.execute :as transforms.execute][metabase.transforms.freshness :as freshness][metabase.transforms.instrumentation :as transforms.instrumentation]@@ -54,6 +55,32 @@transform-id-2])))ordering)))+(defn dependencies->plan+ "Build an execution plan from a `dependencies` map (transform-id -> #{dep-ids}, as returned by+ `transform-ordering`) and the transforms it covers. Returns `{:order [...transforms in execution+ order] :deps dependencies}`. Throws if the dependency graph contains a cycle.++ This is the pure, in-memory part of planning — no DB or `table-dependencies` work — so callers+ that already hold a dependency map (e.g. a DAG reprocess that computed the whole graph once) can+ build a plan without recomputing it."+ [dependencies all-transforms]+ (let [transforms-by-id (into {}+ (keep (fn [{:keys [id] :as transform}]+ (when (contains? dependencies id)+ [id transform])))+ all-transforms)+ sorted-ord (sorted-ordering dependencies transforms-by-id)]+ (when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]+ (let [id->name (into {} (map (juxt :id :name)) all-transforms)]+ (throw (ex-info (str "Cyclic transform definitions detected: "+ (str/join " → " (map id->name cycle)))+ {:cycle cycle}))))+ (loop [complete (ordered-set/ordered-set)]+ (if-let [current-transform (next-transform sorted-ord transforms-by-id complete)]+ (recur (conj complete (:id current-transform)))+ {:order (map transforms-by-id complete)+ :deps dependencies}))))+(defn- get-plan [transform-ids](tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}(let [all-transforms (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies])@@ -73,20 +100,10 @@;; Lazily backfill the table_dependencies column for any closure transform we had to compute live.(transforms-base.ordering/persist-table-dependencies! uncached);; Fetch full rows only for the closure, which is what callers actually consume.- (let [transforms-by-id (if (seq dependencies)- (u/index-by :id (t2/select :model/Transform :id [:in (keys dependencies)]))- {})- sorted-ord (sorted-ordering dependencies transforms-by-id)]- (when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]- (let [id->name (into {} (map (juxt :id :name)) (vals transforms-by-id))]- (throw (ex-info (str "Cyclic transform definitions detected: "- (str/join " → " (map id->name cycle)))- {:cycle cycle}))))- (loop [complete (ordered-set/ordered-set)]- (if-let [current-transform (next-transform sorted-ord transforms-by-id complete)]- (recur (conj complete (:id current-transform)))- {:order (map transforms-by-id complete)- :deps dependencies}))))))+ (let [closure-transforms (if (seq dependencies)+ (t2/select :model/Transform :id [:in (keys dependencies)])+ [])]+ (dependencies->plan dependencies closure-transforms)))))(defn- wait-for-transform-slot!"Poll until no active run exists for `transform-id`, up to `timeout-ms`."@@ -97,7 +114,13 @@(>= (u/since-ms timer) timeout-ms) false:else (do (Thread/sleep 2000) (recur)))))-(defn- run-transform! [run-id run-method user-id started-run-id {transform-id :id :as transform}]+(defn- run-transform!+ "Execute a single transform as part of a coordinated run. `ctx` is the coordinator context; this+ uses its `:parent-run` (a `[type id]` tuple — `[:job id]`/`[:dag id]` — that links the created+ `transform_run` row to its coordinating `transform_job_run`/`transform_dag_run`), `:run-method`,+ `:user-id`, and `:add-run-activity!` (a zero-arg fn called after successful execution to touch the+ coordinating run's updated_at)."+ [{:keys [parent-run run-method user-id add-run-activity!]} started-run-id {transform-id :id :as transform}](cond(not (transforms.u/check-feature-enabled transform))(log/warnf "Skip running transform %d due to lacking premium features" transform-id)@@ -118,10 +141,10 @@(pr-str transform-id) (transforms.settings/transform-timeout)){:transform-id transform-id :error :already-running-timeout}))(let [result (try- (log/info "Executing job transform" (pr-str transform-id))+ (log/info "Executing transform" (pr-str transform-id))(transforms.execute/execute! transform {:run-method run-method:user-id user-id- :job-run-id run-id+ :parent-run parent-run;; lets the coordinator cancel exactly the run;; this worker started (see [[cancel-worker!]]):on-start #(deliver started-run-id %)})@@ -133,7 +156,7 @@(throw e))))](when (= :already-running result)(recur))))))- (transforms.job-run/add-run-activity! run-id))))+ (add-run-activity!))))(defn- lane-for"Lane a transform runs in: `:py` for python transforms (single-slot), `:sql` otherwise."@@ -149,18 +172,20 @@"Run `transform` on its own thread; returns a future yielding a `::status` completion map.`started-run-id` is a promise the worker delivers its transform run id to once `execute!` hascreated it — at most once, since the `:already-running` retry only re-enters after a failed- start — so [[cancel-worker!]] can target exactly that run."- [run-id run-method user-id started-run-id transform]+ start — so [[cancel-worker!]] can target exactly that run. `ctx` is the coordinator context,+ forwarded to [[run-transform!]] "+ [{[run-type run-id] :parent-run :as ctx} started-run-id transform](future- (try- (run-transform! run-id run-method user-id started-run-id transform)- {::status :succeeded ::transform transform}- (catch Throwable t- (log/errorf t "Transform %s in run %s failed" (pr-str (:id transform)) (pr-str run-id))- {::status :failed- ::transform transform- ::message (or (ex-message t) (str t))- ::throwable t}))))+ (log/with-context {:run-type run-type, :run-id run-id, :transform-id (:id transform)}+ (try+ (run-transform! ctx started-run-id transform)+ {::status :succeeded ::transform transform}+ (catch Throwable t+ (log/errorf t "Transform %s in run %s failed" (pr-str (:id transform)) (pr-str run-id))+ {::status :failed+ ::transform transform+ ::message (or (ex-message t) (str t))+ ::throwable t})))))(defn- transform-target-key"A coarse identity for a transform's output table, used to keep two transforms that write the@@ -173,7 +198,7 @@(defn- dispatch-ready!"Submit every transform whose deps have all succeeded and whose lane has a free slot, recording thetransitive cascade of dep-failures along the way. Returns the updated state."- [st {:keys [plan deps lanes run-id run-method user-id]}]+ [st {:keys [plan deps lanes] :as ctx}](reduce(fn [{:keys [succeeded failed in-flight-targets] :as st} t](let [id (:id t)@@ -203,7 +228,7 @@(and (every? succeeded dep-ids)(< (count in-flight-now) (lanes lane-key)))(let [started-run-id (promise)- fut (submit-transform! run-id run-method user-id started-run-id t)]+ fut (submit-transform! ctx started-run-id t)](cond-> (assoc-in st [:in-flight lane-key id]{:future fut:timer (u/start-timer)@@ -270,19 +295,14 @@(log/warnf t "Error canceling in-flight worker for transform %s" (pr-str id))))))(defonce ^:private active-runs- ;; job-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped)+ ;; job-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped).(atom {}))(defn- heartbeat-and-reconcile-runs!- "Stamp a heartbeat on every job run this process is coordinating, then deliver the `gone` promise- of any that another path (reaper, force-fail) already terminated, so its coordinator aborts."+ "Stamp a heartbeat on every active job run this process owns, then deliver the `gone` promise of+ any that were terminated externally so their coordinator aborts."[]- (rt/heartbeat-and-reconcile! {:model :model/TransformJobRun- :active [:= :is_active true]- :ids (keys @active-runs)- :heartbeat! transforms.job-run/heartbeat-runs!- :on-gone (fn [run-id]- (some-> (get @active-runs run-id) (deliver true)))}))+ (coordinated-run/heartbeat-and-reconcile! :model/TransformJobRun active-runs))(defn- run-coordinator-loop!"Dispatch ready transforms; then each round sweep the in-flight workers and refill freed slots —@@ -317,26 +337,34 @@(:now (t2/query-one {:select [[[:raw "current_timestamp"] :now]]})))(defn run-transforms!- "Run `transform-ids-to-run` and their dependencies, honoring the DAG.+ "Run the transforms of `plan`, honoring the DAG.Each transform runs on its own thread; SQL transforms run concurrently up to`transform-run-job-sql-concurrency`, python transforms one at a time (the python-runner service hasa single worker). A failed dependency fails its dependents transitively; a worker that overruns thetransform timeout (plus grace) is interrupted and failed.- Transforms pulled into the plan only as dependencies (not directly requested) are skipped while- still fresh, unless `skip-fresh-deps?` is false.+ `transform-ids-to-run` are the directly requested transforms; the rest of the plan was pulled in+ as dependencies and is skipped while still fresh, unless `skip-fresh-deps?` is false.++ `:parent-run-type` (`:job`/`:dag`, default `:job`) selects which coordinating run each member+ `transform_run` links back to.+ `:active-runs-atom` (default `active-runs`) is the atom that tracks coordinator liveness for heartbeating.+ `:add-run-activity!` is a zero-arg fn called after each successful transform execution.Returns `{::status :succeeded}`, `{::status :failed ::failures [...]}`, or `{::status :aborted}`- when the job run was terminated externally (e.g. reaped) while this coordinator was still running."- [run-id transform-ids-to-run {:keys [run-method start-promise user-id skip-fresh-deps?]- :or {skip-fresh-deps? true}}]+ when the coordinating run was terminated externally (e.g. reaped) while this coordinator was still running."+ [run-id transform-ids-to-run {plan :order deps :deps} {:keys [run-method user-id skip-fresh-deps?+ parent-run-type active-runs-atom add-run-activity!]+ :or {skip-fresh-deps? true+ parent-run-type :job+ active-runs-atom active-runs+ add-run-activity! (constantly nil)}}](let [gone (promise)- _ (swap! active-runs assoc run-id gone)+ _ (swap! active-runs-atom assoc run-id gone)final-state(try- (let [{plan :order deps :deps} (get-plan transform-ids-to-run)- requested (set transform-ids-to-run)+ (let [requested (set transform-ids-to-run)closure (into #{} (map :id) plan);; Only deps pulled into the plan (not directly requested) are freshness-gated. Seeding them;; as :succeeded lets their dependents dispatch while they themselves are never submitted.@@ -350,23 +378,23 @@:in-flight {:sql {} :py {}};; target tables currently being written by an in-flight transform:in-flight-targets #{}}- ctx {:plan plan- :deps deps- :run-id run-id- :run-gone? #(realized? gone)- :run-method run-method- :user-id user-id+ ctx {:plan plan+ :deps deps+ :parent-run [parent-run-type run-id]+ :run-gone? #(realized? gone)+ :run-method run-method+ :user-id user-id+ :add-run-activity! add-run-activity!;; lane -> max concurrent workers- :lanes {:sql (max 1 (transforms.settings/transform-run-job-sql-concurrency))- :py 1}- :timeout-ms (+ (u/minutes->ms (transforms.settings/transform-timeout))- transform-worker-grace-ms)}]+ :lanes {:sql (max 1 (transforms.settings/transform-run-job-sql-concurrency))+ :py 1}+ :timeout-ms (+ (u/minutes->ms (transforms.settings/transform-timeout))+ transform-worker-grace-ms)}](when (seq skip)(log/infof "Skipping %d fresh pulled-in dependency transform(s): %s" (count skip) (pr-str skip)))- (when start-promise (deliver start-promise :started))(run-coordinator-loop! init-state ctx))(finally- (swap! active-runs dissoc run-id)))]+ (swap! active-runs-atom dissoc run-id)))](cond(:aborted? final-state) {::status :aborted}(seq (:failures final-state)) {::status :failed ::failures (:failures final-state)}@@ -395,7 +423,10 @@:scheduled (contains? scheduled id))))plan)))-(defn- compile-transform-failure-messages [failures]+(defn- compile-transform-failure-messages+ "Render the `::failures` of a coordinated run (as returned by [[run-transforms!]]) into a single+ human-readable message, one block per failed transform with a link to its run."+ [failures](->> failures(map (fn [failure](format "%s %s:\n%s"@@ -404,6 +435,34 @@(::message failure))))(str/join "\n\n")))+(defn execute-coordinated-run!+ "Run `transform-ids` per `plan` (as returned by [[get-plan]] or [[dependencies->plan]]) as the+ members of the coordinated run `run-id` (a row of `model`) and move the row to its terminal+ state, failing it if execution throws (the throwable is rethrown). `label` names the run in+ logs; `run-opts` are passed through to [[run-transforms!]]. Returns the [[run-transforms!]]+ result so callers can act on failures."+ [model run-id transform-ids plan label run-opts]+ (try+ (let [result (run-transforms! run-id transform-ids plan+ (assoc run-opts+ :add-run-activity! #(coordinated-run/add-run-activity! model run-id)))]+ (case (::status result)+ :succeeded (coordinated-run/succeed-started-run! model run-id)+ ;; terminated externally (e.g. reaped): the row is already terminal, just log+ :aborted (log/warnf "%s %s was terminated externally; coordinator aborted." label (pr-str run-id))+ :failed (try+ (coordinated-run/fail-started-run! model run-id+ {:message (compile-transform-failure-messages (::failures result))})+ (catch Exception e+ (log/errorf e "Error when failing %s %s." label (pr-str run-id)))))+ result)+ (catch Throwable t+ (try+ (coordinated-run/fail-started-run! model run-id {:message (ex-message t)})+ (catch Exception e+ (log/errorf e "Error when failing %s." label)))+ (throw t))))+(defn- active-users-to-edit-transform"Return the users that edited the transform, in reverse chron.Only returns active users.@@ -467,43 +526,61 @@failures)}))))(defn run-job!- "Runs all transforms for a given job and their dependencies."- [job-id {:keys [run-method] :as opts}]- (if (transforms.job-run/running-run-for-job-id job-id)- (log/info "Not executing transform job" (pr-str job-id) "because it is already running")- (let [transforms (job-transform-ids job-id)]- (if (empty? transforms)- (log/info "Skipping transform job" (pr-str job-id) "because it has no transforms to run")- (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]- (tracing/with-span :tasks "task.transform.run-job" {:transform.job/id job-id- :transform.job/run-method (name run-method)- :transform.job/count (count transforms)}- (transforms.instrumentation/with-job-timing [job-id run-method]- (try ;; catch any catastrophic problems- (let [result (run-transforms! run-id transforms opts)]- (case (::status result)- :succeeded (transforms.job-run/succeed-started-run! run-id)- ;; terminated externally (e.g. reaped): the row is already terminal and the- ;; terminator already notified, so just log- :aborted (log/warnf "Transform job run %s for job %s was terminated externally; coordinator aborted."- (pr-str run-id) (pr-str job-id))- :failed (try- (transforms.job-run/fail-started-run! run-id {:message (compile-transform-failure-messages (::failures result))})- (when (= :cron run-method)- (notify-transform-failures job-id (::failures result)))- (catch Exception e- (log/error e "Error when failing a transform run.")))))- (catch Throwable t- ;; We don't expect a catastrophic failure, but neither did the Titanic.- (try- (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})- (when (and (::transform-failure (ex-data t))- (= :cron run-method)) ;; Catastrophic job failures are included in the digest- (notify-transform-failures job-id (::failures (ex-data t))))- (catch Exception e- (log/error e "Error when failing a transform job run.")))- (throw t)))))- run-id)))))+ "Runs all transforms for a given job and their dependencies.++ `:start-promise`, if provided, is delivered `[:started run-id]` once the run row is created, `nil`+ if nothing was run (already running / no transforms), or a Throwable on a pre-start failure.+ Returns the run id, or nil if nothing was executed."+ [job-id {:keys [run-method start-promise] :as opts}]+ (try+ (if (transforms.job-run/running-run-for-job-id job-id)+ (do (log/info "Not executing transform job" (pr-str job-id) "because it is already running")+ (some-> start-promise (deliver nil))+ nil)+ (let [transforms (job-transform-ids job-id)]+ (if (empty? transforms)+ (do (log/info "Skipping transform job" (pr-str job-id) "because it has no transforms to run")+ (some-> start-promise (deliver nil))+ nil)+ (let [plan (get-plan transforms)+ {run-id :id} (transforms.job-run/start-run! job-id run-method)+ ;; failed cron runs are included in the failure digest+ notify! (fn [failures]+ (when (= :cron run-method)+ (try+ (notify-transform-failures job-id failures)+ (catch Exception e+ (log/errorf e "Error notifying failures of transform job %s" (pr-str job-id))))))]+ (some-> start-promise (deliver [:started run-id]))+ (tracing/with-span :tasks "task.transform.run-job" {:transform.job/id job-id+ :transform.job/run-method (name run-method)+ :transform.job/count (count transforms)}+ (transforms.instrumentation/with-job-timing [job-id run-method]+ (try+ (let [result (execute-coordinated-run! :model/TransformJobRun run-id transforms plan+ (format "Transform job run for job %s" (pr-str job-id))+ (-> opts+ (dissoc :start-promise)+ (assoc :parent-run-type :job :active-runs-atom active-runs)))]+ (when (= (::status result) :failed)+ (notify! (::failures result))))+ (catch Throwable t+ (when (::transform-failure (ex-data t))+ (notify! (::failures (ex-data t))))+ (throw t)))))+ run-id))))+ (catch Throwable t+ ;; unblock any caller waiting on the promise on a pre-start failure+ (some-> start-promise (deliver t))+ (throw t))))++(defn cancel-job-run!+ "Cancel an in-progress job run and request cancellation of its still-running member transform runs.+ Returns true if the run was active and is now canceled, false if it had already finished."+ [run-id]+ (coordinated-run/cancel! :model/TransformJobRun :job_run_id run-id))++;;; ------------------------------------------- Orphan reaping -------------------------------------------(defn- reap-orphaned-runs!"Reap job runs whose coordinator process died (stale heartbeat)."diff --git a/src/metabase/transforms/models/dag_run.clj b/src/metabase/transforms/models/dag_run.cljnew file mode 100644index 00000000000..6d3edb6c89a--- /dev/null+++ b/src/metabase/transforms/models/dag_run.clj@@ -0,0 +1,53 @@+(ns metabase.transforms.models.dag-run+ "A `transform_dag_run` tracks a manual DAG-reprocess run: reprocessing the transitive dependency DAG+ rooted at a single seed transform (upstream or downstream). Like a scheduled job run it coordinates+ many member transform runs, but it is always triggered manually from one transform rather than by a+ job — so it has its own table rather than overloading `transform_job_run`. Member runs link back+ via `transform_run.dag_run_id`."+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(methodical/defmethod t2/table-name :model/TransformDagRun [_model] :transform_dag_run)++(derive :model/TransformDagRun :metabase/model)+(derive :model/TransformDagRun :hook/timestamped?)++(t2/deftransforms :model/TransformDagRun+ {:status mi/transform-keyword+ :direction mi/transform-keyword})++(defn start-dag-run!+ "Start a DAG-reprocess run seeded from `source-transform-id`, traversing `direction`+ (`:upstream`/`:downstream`). `user-id` is the user who triggered it; `transform-count` is the+ number of transforms the run's plan selected to run, stored for display in run listings.+ Snapshots the seed transform's name and entity_id so the run stays displayable after the+ transform is deleted (its FK is SET NULL, as for `transform_run.transform_id`)."+ [source-transform-id direction user-id transform-count]+ (let [transform (t2/select-one [:model/Transform :name :entity_id] :id source-transform-id)]+ (t2/insert-returning-instance! :model/TransformDagRun+ {:source_transform_id source-transform-id+ :source_transform_name (:name transform)+ :source_transform_entity_id (:entity_id transform)+ :direction direction+ :transform_count transform-count+ :user_id user-id+ :status :started+ :is_active true})))++(defn running-run-for-source-transform-id+ "Return the single active DAG run seeded from `source-transform-id`, or nil."+ [source-transform-id]+ (t2/select-one :model/TransformDagRun+ :source_transform_id source-transform-id+ :is_active true))++(defn transform-runs-for-dag-run+ "Return the transform runs that were part of the given DAG run, ordered by start time."+ [dag-run-id]+ (t2/select :model/TransformRun+ {:where [:= :dag_run_id dag-run-id]+ :order-by [[:start_time :asc]]}))diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.cljindex ef107d4dd3e..263afdfbb6b 100644--- a/src/metabase/transforms/models/job_run.clj+++ b/src/metabase/transforms/models/job_run.clj@@ -1,7 +1,7 @@(ns metabase.transforms.models.job-run(:require[metabase.models.interface :as mi]- [metabase.run-tracking.core :as rt]+ [metabase.transforms.coordinated-run :as coordinated-run][metabase.transforms.models.util :as transforms.models.u][methodical.core :as methodical][toucan2.core :as t2]@@ -36,65 +36,25 @@(t2/reducible-select :model/TransformJobRun (latest-runs-query job-ids)))))(defn start-run!- "Start a run"+ "Start a run. Snapshots the job's name and entity_id so the run stays displayable after the job+ is deleted (`transform_job_run.job_id` has no FK; job runs outlive their job)."([job-id run-method]- (t2/insert-returning-instance! :model/TransformJobRun- {:job_id job-id- :run_method run-method- :status :started- :is_active true})))--(defn add-run-activity!- "Notes that a run has had activity"- [run-id]- (t2/update! :model/TransformJobRun- :id run-id- :is_active true- {:updated_at :%now}))--(defn succeed-started-run!- "Mark a started run as successfully completed."- ([run-id]- (succeed-started-run! run-id {}))- ([run-id properties]- (t2/update! :model/TransformJobRun- :id run-id- :is_active true- (merge {:end_time :%now}- properties- {:status :succeeded- :is_active nil}))))--(defn fail-started-run!- "Mark the started active run as failed and inactive."- [run-id properties]- (t2/update! :model/TransformJobRun- :id run-id- :is_active true- (merge {:end_time :%now}- properties- {:status :failed- :is_active nil})))--(defn heartbeat-runs!- "Stamp `last_heartbeat = now` on the given still-active job-run-ids."- [run-ids]- (rt/heartbeat-ids! :model/TransformJobRun [:= :is_active true] :last_heartbeat run-ids))+ ;; :built_in_type so the after-select hook localizes built-in job names; str realizes the+ ;; LocalizedString into the snapshot+ (let [job (t2/select-one [:model/TransformJob :name :entity_id :built_in_type] :id job-id)]+ (t2/insert-returning-instance! :model/TransformJobRun+ {:job_id job-id+ :job_name (some-> (:name job) str)+ :job_entity_id (:entity_id job)+ :run_method run-method+ :status :started+ :is_active true}))))(defn reap-orphaned-runs!"Time out active job runs whose `last_heartbeat` is older than `stale-minutes` (their coordinatorprocess is presumed dead). Returns the rows that were timed out so callers can notify."[stale-minutes]- (rt/reap-orphaned!- {:model :model/TransformJobRun- :active [:= :is_active true]- :stale [:< :last_heartbeat (rt/cutoff stale-minutes :minute)]- :terminal {:status "timeout" :end_time :%now :is_active nil :message "Timed out: crashed"}- :metrics {:total-metric :metabase-transforms/timeouts-total- :latency-metric :metabase-transforms/timeout-detection-latency-ms- :tags {:type "job"}- :latency-column :last_heartbeat- :timeout-ms (rt/unit->ms stale-minutes :minute)}}))+ (coordinated-run/reap-orphaned-runs! :model/TransformJobRun "job" stale-minutes))(defn running-run-for-job-id"Return a single active job run or nil."@@ -107,32 +67,24 @@"Return a page of the list of job runs.Follows the conventions used by the FE."- [{:keys [offset limit sort-column sort-direction job-id status run-method start-time]}]- (let [offset (or offset 0)- limit (or limit 20)- sort-direction (or (keyword sort-direction) :desc)- nulls-sort (if (= sort-direction :asc) :nulls-last :nulls-first)- sort-column (keyword sort-column)- order-by (case sort-column- :start_time [[sort-column sort-direction]]- :end_time [[sort-column sort-direction nulls-sort]]- [[:start_time sort-direction]- [:end_time sort-direction nulls-sort]])- where-cond (cond-> []- job-id (conj [:= :job_id job-id])- status (conj [:= :status status])- (= status "started") (conj [:= :is_active true])- run-method (conj [:= :run_method run-method])- start-time (conj (transforms.models.u/timestamp-constraint :start_time start-time)))- where (when (seq where-cond) (into [:and] where-cond))- query-opts (cond-> {:order-by order-by :offset offset :limit limit}- where (assoc :where where))- count-opts (if where {:where where} {})- runs (t2/select :model/TransformJobRun query-opts)]- {:data runs+ [{:keys [sort-column sort-direction job-id status run-method start-time offset limit]}]+ (let [offset (or offset 0)+ limit (or limit 20)+ where-cond (cond-> []+ job-id (conj [:= :job_id job-id])+ status (conj [:= :status status])+ (= status "started") (conj [:= :is_active true])+ run-method (conj [:= :run_method run-method])+ start-time (conj (transforms.models.u/timestamp-constraint :start_time start-time)))+ where (when (seq where-cond) (into [:and] where-cond))]+ {:data (t2/select :model/TransformJobRun+ (cond-> {:order-by (transforms.models.u/run-order-by sort-column sort-direction)+ :offset offset+ :limit limit}+ where (assoc :where where))):limit limit:offset offset- :total (t2/count :model/TransformJobRun count-opts)}))+ :total (t2/count :model/TransformJobRun (if where {:where where} {}))}))(defn transform-runs-for-job-run"Return transform runs that were part of the given job run, ordered by start time."diff --git a/src/metabase/transforms/models/util.clj b/src/metabase/transforms/models/util.cljindex 04846a9eefc..9dfdc573c33 100644--- a/src/metabase/transforms/models/util.clj+++ b/src/metabase/transforms/models/util.clj@@ -22,3 +22,17 @@(into [:and] (remove nil?)[(when start [:>= field-name start])(when end [:< field-name end])])))++(defn run-order-by+ "Standard `:order-by` clause for a paged run listing. Sorts by `sort-column` (`:start_time` or+ `:end_time`; anything else — including nil — falls back to ordering by start_time then end_time)+ in `sort-direction` (`:asc`/`:desc`, defaulting to `:desc`), with in-progress rows (null+ `end_time`) always ordered last."+ [sort-column sort-direction]+ (let [sort-direction (or (keyword sort-direction) :desc)+ nulls-sort (if (= sort-direction :asc) :nulls-last :nulls-first)]+ (case (keyword sort-column)+ :start_time [[:start_time sort-direction]]+ :end_time [[:end_time sort-direction nulls-sort]]+ [[:start_time sort-direction]+ [:end_time sort-direction nulls-sort]])))diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex 65fde1978a7..9efc6fbf2c6 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -32,7 +32,7 @@(defn- run-mbql-transform!([transform] (run-mbql-transform! transform nil))([{:keys [id source target owner_user_id creator_id] :as transform}- {:keys [run-method on-start user-id job-run-id]}]+ {:keys [run-method on-start user-id parent-run]}];; `:target` is already workspace-rewritten — `resolve-transform-target` runs in;; `metabase.transforms.execute/execute!` before dispatch.(try@@ -42,7 +42,8 @@run-user-id (if (and (= run-method :manual) user-id)user-id(or owner_user_id creator_id))- {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id :job-run-id job-run-id)]+ {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id+ :parent-run parent-run)](when on-start (on-start run-id))(driver.conn/with-write-connection(log/info "Executing transform" id "with target" (pr-str target)diff --git a/src/metabase/transforms/run_listing.clj b/src/metabase/transforms/run_listing.cljnew file mode 100644index 00000000000..756c4ce34c6--- /dev/null+++ b/src/metabase/transforms/run_listing.clj@@ -0,0 +1,153 @@+(ns metabase.transforms.run-listing+ "A single paged listing over the three kinds of root run:++ - a job run (`transform_job_run`)+ - a manual DAG-reprocess run (`transform_dag_run`)+ - a standalone transform run (`transform_run` not belonging to a job or DAG run)++ Member transform runs that belong to a job or DAG run are excluded — they are reachable through+ their parent run. Because ids are per-table, a row is identified by the `(run_type, id)` pair;+ `entity_id` is the id of the associated job/transform."+ (:require+ [medley.core :as m]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.models.util :as transforms.models.u]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def run-types+ "The `run_type` discriminator values a listing row can have."+ [:job :dag :transform])++;;; ---------------------------------------------- Per-source subqueries ----------------------------------------------++;; Each branch must project the same columns in the same order for the UNION ALL to line up;+;; `[nil :col]` fills in columns a table lacks.++(defn- job-run-subquery [transform-ids]+ {:select [[[:inline "job"] :run_type]+ :id+ [:job_id :entity_id]+ [:job_name :entity_name]+ [nil :direction]+ [nil :transform_count]+ :run_method+ :status :is_active :start_time :end_time :message+ [nil :user_id]]+ :from [:transform_job_run]+ :where (if (seq transform-ids)+ ;; only job runs that actually ran one of these transforms+ [:exists {:select [[[:inline 1]]]+ :from [[:transform_run :member]]+ :where [:and+ [:= :member.job_run_id :transform_job_run.id]+ [:in :member.transform_id transform-ids]]}]+ true)})++(defn- dag-run-subquery [transform-ids]+ {:select [[[:inline "dag"] :run_type]+ :id+ [:source_transform_id :entity_id]+ [:source_transform_name :entity_name]+ :direction+ :transform_count+ [[:inline "manual"] :run_method]+ :status :is_active :start_time :end_time :message+ :user_id]+ :from [:transform_dag_run]+ :where (if (seq transform-ids)+ [:exists {:select [[[:inline 1]]]+ :from [[:transform_run :member]]+ :where [:and+ [:= :member.dag_run_id :transform_dag_run.id]+ [:in :member.transform_id transform-ids]]}]+ true)})++(defn- transform-run-subquery [transform-ids]+ {:select [[[:inline "transform"] :run_type]+ :id+ [:transform_id :entity_id]+ [:transform_name :entity_name]+ [nil :direction]+ [nil :transform_count]+ :run_method+ :status :is_active :start_time :end_time :message+ :user_id]+ :from [:transform_run]+ ;; standalone runs only: those not coordinated by a job or DAG run+ :where (cond-> [:and+ [:= :job_run_id nil]+ [:= :dag_run_id nil]]+ (seq transform-ids) (conj [:in :transform_id transform-ids]))})++(defn- union-subquery+ "The UNION ALL of the branches selected by `types` (a subset of [[run-types]]), each optionally+ narrowed to runs touching one of `transform-ids`."+ [types transform-ids]+ (let [types (set (or (seq types) run-types))]+ {:union-all (cond-> []+ (:job types) (conj (job-run-subquery transform-ids))+ (:dag types) (conj (dag-run-subquery transform-ids))+ (:transform types) (conj (transform-run-subquery transform-ids)))}))++(defn paged-run-summaries+ "Return a page of root runs as a `{:data :limit :offset :total}` envelope. Rows are raw — see+ [[present-run-summaries]].++ Options (filter semantics match [[metabase.transforms.models.transform-run/paged-runs]]):+ - `:types` subset of [[run-types]] to include; nil/empty means all three+ - `:statuses` match any of these status strings+ - `:run-methods` match any of these triggers (`\"manual\"`/`\"cron\"`)+ - `:start-time` date range string (as in the QP date parameters) constraining `start_time`+ - `:end-time` likewise for `end_time`+ - `:transform-ids` only runs that ran any of these transforms (job/DAG runs whose members include+ one, or the standalone runs of them)+ - `:sort-column` `\"start_time\"` / `\"end_time\"`+ - `:sort-direction` `\"asc\"` / `\"desc\"`+ - `:offset`/`:limit` pagination (default 0 / 20)"+ [{:keys [types statuses run-methods start-time end-time transform-ids sort-column sort-direction offset limit]}]+ (let [offset (or offset 0)+ limit (or limit 20)+ inner (union-subquery types transform-ids)+ where (into [:and] (remove nil?)+ [(when (seq statuses) [:in :status (set statuses)])+ ;; started ⇒ still active, as in the per-table run listings+ (when (= (set statuses) #{"started"}) [:= :is_active true])+ (when (seq run-methods) [:in :run_method (set run-methods)])+ (when start-time (transforms.models.u/timestamp-constraint :start_time start-time))+ (when end-time (transforms.models.u/timestamp-constraint :end_time end-time))])+ where (when (> (count where) 1) where)+ base (cond-> {:from [[inner :runs]]}+ where (assoc :where where))]+ {:data (t2/query (merge base+ {:select [:*]+ :order-by (transforms.models.u/run-order-by sort-column sort-direction)+ :limit limit+ :offset offset}))+ :limit limit+ :offset offset+ :total (:count (first (t2/query (merge base {:select [[[:count :*] :count]]}))))}))++(defn present-run-summaries+ "Prepare raw summary rows for an API response: hydrate each row's `:name` (nil when the underlying+ job/transform was deleted), keywordize the discriminator columns, and localize timestamps."+ [rows]+ (let [by-type (group-by :run_type rows)+ job-ids (seq (keep :entity_id (get by-type "job")))+ transform-ids (seq (keep :entity_id (concat (get by-type "dag") (get by-type "transform"))))+ job->name (when job-ids (t2/select-pk->fn :name :model/TransformJob :id [:in job-ids]))+ transform->name (when transform-ids (t2/select-pk->fn :name :model/Transform :id [:in transform-ids]))]+ (map (fn [{:keys [run_type entity_id entity_name] :as row}]+ (-> row+ (assoc :name (or (if (= run_type "job")+ (get job->name entity_id)+ (get transform->name entity_id))+ entity_name))+ (dissoc :entity_name)+ (update :run_type keyword)+ (update :status keyword)+ (m/update-existing :run_method #(some-> % keyword))+ (m/update-existing :direction #(some-> % keyword))+ transforms-base.u/localize-run-timestamps))+ rows)))diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex 08962eb05ba..442c42db2da 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -104,7 +104,7 @@;;; ------------------------------------------------- Scheduled Execution --------------------------------------------------(defn- duplicate-key-violation?+(defn duplicate-key-violation?"Check if an exception is a duplicate key violation.Returns true for Postgres, MySQL/MariaDB, and H2 duplicate key errors."[e]@@ -117,19 +117,22 @@"Start a transform run. Throws ex-info with {:error :already-running} if anotherrun is already active (duplicate key violation). Other errors are rethrown.If `user-id` is provided, it will be stored with the run for attribution purposes.- If `job-run-id` is provided, it will be stored with the run to link it to its parent job run."- [id run-method user-id & {:keys [job-run-id]}]- (try- (transform-run/start-run! id (cond-> {:run_method run-method}- user-id (assoc :user_id user-id)- job-run-id (assoc :job_run_id job-run-id)))- (catch Exception e- (if (duplicate-key-violation? e)- (throw (ex-info "Transform is already running"- {:error :already-running- :transform-id id}- e))- (throw e)))))+ `parent-run`, when provided, links the run to its coordinating run as a `[type id]` tuple:+ `[:job id]` is stored in `job_run_id`, `[:dag id]` in `dag_run_id`."+ [id run-method user-id & {:keys [parent-run]}]+ (let [[parent-type parent-id] parent-run]+ (try+ (transform-run/start-run! id (cond-> {:run_method run-method}+ user-id (assoc :user_id user-id)+ (= parent-type :job) (assoc :job_run_id parent-id)+ (= parent-type :dag) (assoc :dag_run_id parent-id)))+ (catch Exception e+ (if (duplicate-key-violation? e)+ (throw (ex-info "Transform is already running"+ {:error :already-running+ :transform-id id}+ e))+ (throw e))))))(defn run-cancelable-transform!"Execute a transform with cancellation support and proper error handling.diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 6eea839638e..3e71ab33473 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -6,16 +6,16 @@[metabase.api.util.handlers :as handlers][metabase.request.core :as request][metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms-rest.api.transform-dag-run :as transforms.dag-run][metabase.transforms-rest.api.transform-job][metabase.transforms-rest.api.transform-tag]+ [metabase.transforms-rest.api.util :as transforms-rest.api.u][metabase.transforms.core :as transforms.core][metabase.transforms.schema :as transforms.schema][metabase.transforms.util :as transforms.u]- [metabase.util.i18n :refer [deferred-tru]]- [metabase.util.jvm :as u.jvm]+ [metabase.util.i18n :refer [deferred-tru LocalizedString]][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]- [ring.util.response :as response][toucan2.core :as t2])(:import(java.time OffsetDateTime)))@@ -68,6 +68,7 @@[:transform_name {:optional true} [:maybe :string]][:transform_entity_id {:optional true} [:maybe :string]][:job_run_id {:optional true} [:maybe pos-int?]]+ [:dag_run_id {:optional true} [:maybe pos-int?]][:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]][:checkpoint_lo_value {:optional true} [:maybe :string]][:checkpoint_hi_value {:optional true} [:maybe :string]]@@ -122,6 +123,7 @@[:transform_name {:optional true} [:maybe :string]][:transform_entity_id {:optional true} [:maybe :string]][:job_run_id {:optional true} [:maybe pos-int?]]+ [:dag_run_id {:optional true} [:maybe pos-int?]][:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]][:checkpoint_lo_value {:optional true} [:maybe :string]][:checkpoint_hi_value {:optional true} [:maybe :string]]@@ -215,7 +217,7 @@[:map[:sort-column {:optional true} [:enum "transform-name" "start-time" "end-time" "status" "run-method" "transform-tags" "duration"]][:sort-direction {:optional true} [:enum "asc" "desc"]]- [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]+ [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]][:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]][:transform-tag-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]][:start-time {:optional true} [:maybe ms/NonBlankString]]@@ -228,6 +230,56 @@:limit (request/limit)))(update :data #(map transforms-base.u/present-run %))))+(def ^:private RunSummaryResponse+ "One row of the unified runs listing: a job run, a manual DAG-reprocess run, or a standalone+ transform run (one not belonging to a job/DAG run). Identified by `(run_type, id)`; `entity_id`+ is the id of the associated job/transform (nil if it was deleted) and `name` its name — live if+ it still exists, otherwise the name snapshotted at run start. `direction` and `transform_count`+ (the number of transforms the run's plan selected to run) are set only for DAG runs."+ [:map {:closed true}+ [:run_type [:enum :job :dag :transform]]+ [:id pos-int?]+ [:entity_id [:maybe pos-int?]]+ [:name [:maybe [:or :string LocalizedString]]]+ [:direction [:maybe [:enum :upstream :downstream]]]+ [:transform_count [:maybe pos-int?]]+ [:run_method [:maybe :keyword]]+ [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]+ [:is_active [:maybe :boolean]]+ [:start_time :any]+ [:end_time {:optional true} [:maybe :any]]+ [:message [:maybe :string]]+ [:user_id [:maybe pos-int?]]])++(api.macros/defendpoint :get "/runs" :- [:map {:closed true}+ [:data [:sequential RunSummaryResponse]]+ [:limit pos-int?]+ [:offset :int]+ [:total :int]]+ "Paginated unified run history: every row is a root run — a job run, a manual DAG-reprocess run,+ or a standalone transform run — never a member run of a job/DAG (those are listed by `GET /run`+ and the per-run `transform-runs` endpoints).++ `types` selects which kinds to include (all by default); `transform-ids` narrows to runs that ran+ any of the given transforms. The remaining filters work as in `GET /run`."+ [_route-params+ query-params :-+ [:map+ [:types {:optional true} [:maybe (ms/QueryVectorOf [:enum "job" "dag" "transform"])]]+ [:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout" "canceled" "canceling"])]]+ [:run-methods {:optional true} [:maybe (ms/QueryVectorOf [:enum "manual" "cron"])]]+ [:start-time {:optional true} [:maybe ms/NonBlankString]]+ [:end-time {:optional true} [:maybe ms/NonBlankString]]+ [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]+ [:sort-column {:optional true} [:maybe [:enum "start_time" "end_time"]]]+ [:sort-direction {:optional true} [:maybe [:enum "asc" "desc"]]]]]+ (api/check-data-analyst)+ (-> (transforms.core/paged-run-summaries (assoc query-params+ :types (map keyword (:types query-params))+ :offset (request/offset)+ :limit (request/limit)))+ (update :data transforms.core/present-run-summaries)))+(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse"Get a transform run by ID."[{:keys [run-id]} :- [:map@@ -290,27 +342,27 @@(t2/update! :model/Transform id {:last_checkpoint_value nil})nil)-(defn run-transform!- "Run a transform. Returns a 202 response with run_id.- The transform must already be fetched and validated."+(defn- check-feature-and-lock!+ "Check that the transform's premium features are enabled and that transforms are not locked by the+ trial quota."[transform](transforms.core/check-feature-enabled! transform)(api/check (not (transforms.core/transform-locked? transform))[402 {:message (deferred-tru "Transforms are temporarily locked because the trial quota has been reached.")- :error-code "metabase_transforms_locked"}])- (let [start-promise (promise)]- (u.jvm/in-virtual-thread*+ :error-code "metabase_transforms_locked"}]))++(defn run-transform!+ "Run a transform. Returns a 202 response with run_id.+ The transform must already be fetched and validated."+ [transform]+ (check-feature-and-lock! transform)+ (transforms-rest.api.u/async-run-response+ (deferred-tru "Transform run started")+ :run_id+ (fn [start-promise](transforms.core/execute! transform {:start-promise start-promise- :run-method :manual- :user-id api/*current-user-id*}))- (when (instance? Throwable @start-promise)- (throw @start-promise))- (let [result @start-promise- run-id (when (and (vector? result) (= (first result) :started))- (second result))]- (-> (response/response {:message (deferred-tru "Transform run started")- :run_id run-id})- (assoc :status 202)))))+ :run-method :manual+ :user-id api/*current-user-id*}))))(api.macros/defendpoint :post "/:id/run" :- [:map[:status [:= 202]]@@ -322,11 +374,53 @@[:id ms/PositiveInt]]](run-transform! (api/read-check :model/Transform id)))+(api.macros/defendpoint :post "/:id/run-dag" :- [:map+ [:status [:= 202]]+ [:body [:map {:closed true}+ [:message :any]+ [:dag_run_id [:maybe pos-int?]]]]]+ "Trigger a DAG-reprocess run starting from a single transform: runs the transform and every+ transform in its transitive dependency closure. Returns a 202 with the created `dag_run_id`, or a+ nil `dag_run_id` when nothing was run (a DAG run for this transform is already in progress, or the+ closure is empty).++ `direction` selects which transforms are included:+ - `upstream` — the seed transform plus all transforms it depends on+ - `downstream` — the seed transform plus all transforms that depend on it"+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ _query-params+ {:keys [direction]} :- [:map+ [:direction (ms/enum-decode-keyword transforms.dag-run/dag-directions)]]]+ (check-feature-and-lock! (api/write-check :model/Transform id))+ (transforms-rest.api.u/async-run-response+ (deferred-tru "DAG run started")+ :dag_run_id+ (fn [start-promise]+ (transforms.core/run-dag! id {:direction direction+ :user-id api/*current-user-id*+ :start-promise start-promise}))))++(api.macros/defendpoint :get "/:id/dag-transforms" :- [:sequential [:map {:closed true}+ [:id pos-int?]+ [:name :string]]]+ "Preview the transforms a DAG reprocess from this transform would run (see `POST /:id/run-dag`),+ in execution order."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ {:keys [direction]} :- [:map [:direction (ms/enum-decode-keyword transforms.dag-run/dag-directions)]]]+ (api/read-check :model/Transform id)+ (mapv (fn [{xform-id :id, xform-name :name}]+ {:id xform-id, :name xform-name})+ (transforms.core/dag-run-transforms id direction)))+(def ^{:arglists '([request respond raise])} routes"`/api/transform` routes."(handlers/routes(api.macros/ns-handler *ns* +auth)))+(def ^{:arglists '([request respond raise])} transform-dag-run-routes+ "`/api/transform-dag-run` routes."+ (api.macros/ns-handler 'metabase.transforms-rest.api.transform-dag-run +auth))+(def ^{:arglists '([request respond raise])} transform-tag-routes"`/api/transform-tag` routes."(api.macros/ns-handler 'metabase.transforms-rest.api.transform-tag +auth))diff --git a/src/metabase/transforms_rest/api/transform_dag_run.clj b/src/metabase/transforms_rest/api/transform_dag_run.cljnew file mode 100644index 00000000000..c79c0268fb2--- /dev/null+++ b/src/metabase/transforms_rest/api/transform_dag_run.clj@@ -0,0 +1,49 @@+(ns metabase.transforms-rest.api.transform-dag-run+ "`/api/transform-dag-run` routes: operations on a single manual DAG-reprocess run — listing its+ member transform runs and canceling it. Triggering and previewing a DAG run live on+ `/api/transform/:id/...` in [[metabase.transforms-rest.api.transform]]."+ (:require+ [metabase.api.common :as api]+ [metabase.api.macros :as api.macros]+ [metabase.api.routes.common :refer [+auth]]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms-rest.api.util :as transforms-rest.api.u]+ [metabase.transforms.core :as transforms.core]+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def dag-directions+ "The DAG traversal directions a reprocess run can take."+ [:upstream :downstream])++(defn- check-seed-transform+ "Check access to a DAG run by applying `check!` (`api/read-check`/`api/write-check`) to its seed+ transform. When the seed was deleted (`source_transform_id` nulled out by the FK) there is no+ transform left to check against, so fall back to requiring the data-analyst role."+ [check! {:keys [source_transform_id] :as _dag-run}]+ (if source_transform_id+ (check! :model/Transform source_transform_id)+ (api/check-data-analyst)))++(api.macros/defendpoint :get "/:run-id/transform-runs" :- [:sequential transforms-rest.api.u/MemberTransformRunResponse]+ "Get the transform runs that made up a specific DAG run."+ [{:keys [run-id]} :- [:map [:run-id ms/PositiveInt]]]+ (let [dag-run (api/check-404 (t2/select-one :model/TransformDagRun :id run-id))]+ (check-seed-transform api/read-check dag-run)+ (->> (t2/hydrate (transforms.core/transform-runs-for-dag-run run-id)+ [:transform :collection :transform_tag_ids])+ (map transforms-base.u/present-run))))++(api.macros/defendpoint :post "/:run-id/cancel" :- :nil+ "Cancel an in-progress manual DAG run and request cancellation of its still-running transforms."+ [{:keys [run-id]} :- [:map [:run-id ms/PositiveInt]]]+ (let [dag-run (api/check-404 (t2/select-one :model/TransformDagRun :id run-id))]+ (check-seed-transform api/write-check dag-run)+ (api/check-400 (transforms.core/cancel-dag-run! run-id))+ nil))++(def ^{:arglists '([request respond raise])} routes+ "`/api/transform-dag-run` routes."+ (api.macros/ns-handler *ns* +auth))diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.cljindex 82ab09d0462..9270b12c3d2 100644--- a/src/metabase/transforms_rest/api/transform_job.clj+++ b/src/metabase/transforms_rest/api/transform_job.clj@@ -7,10 +7,10 @@[metabase.models.interface :as mi][metabase.request.core :as request][metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms-rest.api.util :as transforms-rest.api.u][metabase.transforms.core :as transforms.core][metabase.transforms.util :as transforms.u][metabase.util.i18n :refer [deferred-tru LocalizedString]]- [metabase.util.jvm :as u.jvm][metabase.util.log :as log][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -24,8 +24,10 @@[:map {:closed true}[:id pos-int?][:job_id pos-int?]+ [:job_name {:optional true} [:maybe :string]]+ [:job_entity_id {:optional true} [:maybe :string]][:run_method :keyword]- [:status [:enum :started :succeeded :failed :timeout]]+ [:status [:enum :started :succeeded :failed :timeout :canceled]][:is_active [:maybe :boolean]][:start_time :any][:end_time {:optional true} [:maybe :any]]@@ -216,27 +218,28 @@(transforms.core/delete-job! job-id)api/generic-204-no-content)-(api.macros/defendpoint :post "/:job-id/run" :- [:map {:closed true}- [:message :string]- [:job_run_id :string]]- "Run a transform job manually. By default, fresh pulled-in dependencies are skipped; pass `run_all`- to force-refresh the whole plan."+(api.macros/defendpoint :post "/:job-id/run" :- [:map+ [:status [:= 202]]+ [:body [:map {:closed true}+ [:message :any]+ [:job_run_id [:maybe pos-int?]]]]]+ "Run a transform job manually. Returns a 202 with the created `job_run_id`, or a nil `job_run_id`+ when nothing was run (the job is already running, or has no transforms). By default, fresh+ pulled-in dependencies are skipped; pass `run_all` to force-refresh the whole plan."[{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]_query-params{:keys [run_all]} :- [:map[:run_all {:default false} :boolean]]](log/info "Manual run of transform job" job-id)(api/write-check (t2/select-one :model/TransformJob :id job-id))- (u.jvm/in-virtual-thread*- (try+ (transforms-rest.api.u/async-run-response+ (deferred-tru "Job run started")+ :job_run_id+ (fn [start-promise](transforms.core/run-job! job-id {:run-method :manual:user-id api/*current-user-id*- :skip-fresh-deps? (not run_all)})- (catch Throwable t- (log/error "Error executing transform job" job-id)- (log/error t))))- {:message "Job run started"- :job_run_id (str "stub-" job-id "-" (System/currentTimeMillis))})+ :skip-fresh-deps? (not run_all)+ :start-promise start-promise}))))(api.macros/defendpoint :get "/:job-id" :- TransformJobResponse"Get a transform job by ID."@@ -290,8 +293,10 @@[:map {:closed true}[:id pos-int?][:job_id pos-int?]+ [:job_name {:optional true} [:maybe :string]]+ [:job_entity_id {:optional true} [:maybe :string]][:run_method :keyword]- [:status [:enum :started :succeeded :failed :timeout]]+ [:status [:enum :started :succeeded :failed :timeout :canceled]][:is_active [:maybe :boolean]][:start_time :any][:end_time {:optional true} [:maybe :any]]@@ -299,26 +304,6 @@[:created_at :any][:updated_at :any]])-(def ^:private TransformRunForJobRunResponse- [:map {:closed true}- [:id pos-int?]- [:transform_id [:maybe pos-int?]]- [:job_run_id [:maybe pos-int?]]- [:run_method :keyword]- [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]- [:is_active [:maybe :boolean]]- [:start_time :any]- [:end_time {:optional true} [:maybe :any]]- [:message [:maybe :string]]- [:user_id [:maybe pos-int?]]- [:transform_name {:optional true} [:maybe :string]]- [:transform_entity_id {:optional true} [:maybe :string]]- [:transform {:optional true} [:maybe :map]]- [:metered_as {:optional true} [:maybe :string]]- [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]- [:checkpoint_lo_value {:optional true} [:maybe :string]]- [:checkpoint_hi_value {:optional true} [:maybe :string]]])-(api.macros/defendpoint :get "/:job-id/runs" :- [:map {:closed true}[:data [:sequential JobRunResponse]][:limit pos-int?]@@ -327,7 +312,7 @@"Get paginated run history for a transform job."[{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]query-params :- [:map- [:status {:optional true} [:maybe [:enum "started" "succeeded" "failed" "timeout"]]]+ [:status {:optional true} [:maybe [:enum "started" "succeeded" "failed" "timeout" "canceled"]]][:run-method {:optional true} [:maybe [:enum "manual" "cron"]]][:start-time {:optional true} [:maybe ms/NonBlankString]][:sort-column {:optional true} [:maybe [:enum "start_time" "end_time"]]]@@ -339,7 +324,7 @@:limit (request/limit)))(update :data #(map transforms-base.u/present-run %))))-(api.macros/defendpoint :get "/:job-id/runs/:run-id/transform-runs" :- [:sequential TransformRunForJobRunResponse]+(api.macros/defendpoint :get "/:job-id/runs/:run-id/transform-runs" :- [:sequential transforms-rest.api.u/MemberTransformRunResponse]"Get the transform runs that made up a specific job run."[{:keys [job-id run-id]} :- [:map[:job-id ms/PositiveInt]@@ -351,6 +336,16 @@(->> (t2/hydrate runs [:transform :collection :transform_tag_ids])(map transforms-base.u/present-run))))+(api.macros/defendpoint :post "/:job-id/runs/:run-id/cancel" :- :nil+ "Cancel an in-progress job run and request cancellation of its still-running transforms."+ [{:keys [job-id run-id]} :- [:map+ [:job-id ms/PositiveInt]+ [:run-id ms/PositiveInt]]]+ (api/write-check :model/TransformJob job-id)+ (api/check-404 (t2/select-one :model/TransformJobRun :id run-id :job_id job-id))+ (api/check-400 (transforms.core/cancel-job-run! run-id))+ nil)+(def ^{:arglists '([request respond raise])} routes"`/api/transform-job` routes."(api.macros/ns-handler *ns* +auth))diff --git a/src/metabase/transforms_rest/api/util.clj b/src/metabase/transforms_rest/api/util.cljnew file mode 100644index 00000000000..6c6222f7d62--- /dev/null+++ b/src/metabase/transforms_rest/api/util.clj@@ -0,0 +1,52 @@+(ns metabase.transforms-rest.api.util+ "Helpers shared by the transforms REST API namespaces."+ (:require+ [metabase.util.jvm :as u.jvm]+ [metabase.util.log :as log]+ [ring.util.response :as response]))++(set! *warn-on-reflection* true)++(def MemberTransformRunResponse+ "Response schema for a member transform run of a coordinated (job or DAG) run, hydrated with its+ transform. Exactly one of `job_run_id`/`dag_run_id` is set, naming the coordinating run."+ [:map {:closed true}+ [:id pos-int?]+ [:transform_id [:maybe pos-int?]]+ [:job_run_id [:maybe pos-int?]]+ [:dag_run_id [:maybe pos-int?]]+ [:run_method :keyword]+ [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]+ [:is_active [:maybe :boolean]]+ [:start_time :any]+ [:end_time {:optional true} [:maybe :any]]+ [:message [:maybe :string]]+ [:user_id [:maybe pos-int?]]+ [:transform_name {:optional true} [:maybe :string]]+ [:transform_entity_id {:optional true} [:maybe :string]]+ [:transform {:optional true} [:maybe :map]]+ [:metered_as {:optional true} [:maybe :string]]+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]+ [:checkpoint_lo_value {:optional true} [:maybe :string]]+ [:checkpoint_hi_value {:optional true} [:maybe :string]]])++(defn async-run-response+ "Launch `start!` — a fn of a start-promise — on a virtual thread and respond `202` as soon as the+ run row exists, without waiting for the run to finish. `start!` must deliver the promise+ `[:started run-id]` once the row is created, `nil` when nothing was run, or a Throwable on a+ pre-start failure (rethrown). The response body is `{:message message, <id-key> <run-id-or-nil>}`."+ [message id-key start!]+ (let [start-promise (promise)]+ (u.jvm/in-virtual-thread*+ (try+ (start! start-promise)+ (catch Throwable t+ ;; post-start failures: the run row was already failed by the runner; this is a fallback log+ (log/error t "Async run failed"))))+ (when (instance? Throwable @start-promise)+ (throw @start-promise))+ (let [result @start-promise+ run-id (when (and (vector? result) (= (first result) :started))+ (second result))]+ (-> (response/response {:message message, id-key run-id})+ (assoc :status 202)))))diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex 64fe7e1e734..afee9df4fed 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -74,6 +74,7 @@:model/Transform:model/TransformRun:model/TransformRunCancelation+ :model/TransformDagRun:model/TransformJob:model/TransformJobRun:model/TransformJobTransformTagdiff --git a/test/metabase/transforms/dag_test.clj b/test/metabase/transforms/dag_test.cljnew file mode 100644index 00000000000..821470d5e35--- /dev/null+++ b/test/metabase/transforms/dag_test.clj@@ -0,0 +1,161 @@+(ns metabase.transforms.dag-test+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.transforms.coordinated-run :as coordinated-run]+ [metabase.transforms.dag :as dag]+ [metabase.transforms.models.dag-run :as dag-run]+ [metabase.transforms.models.transform-run :as transform-run]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; ------------------------------------------- Pure graph logic -------------------------------------------++(deftest dependents-graph-test+ (testing "reverses a forward dependency map (reads) into a dependents map (read-by)"+ (is (= {1 #{2 3} 2 #{4} 3 #{4}}+ (#'dag/dependents-graph {2 #{1} 3 #{1} 4 #{2 3} 1 #{}}))))+ (testing "a node nothing depends on is absent from the dependents map"+ (is (= {1 #{2}}+ (#'dag/dependents-graph {2 #{1}}))))+ (testing "empty graph"+ (is (= {} (#'dag/dependents-graph {})))))++(deftest reachable-test+ (let [graph {1 #{2 3} 2 #{4} 3 #{4} 4 #{}}]+ (testing "includes the start node and everything transitively reachable"+ (is (= #{1 2 3 4} (#'dag/reachable graph 1)))+ (is (= #{2 4} (#'dag/reachable graph 2)))+ (is (= #{4} (#'dag/reachable graph 4))))+ (testing "a start node with no outgoing edges reaches only itself"+ (is (= #{5} (#'dag/reachable {} 5)))))+ (testing "cycles terminate (each node visited once)"+ (is (= #{1 2 3} (#'dag/reachable {1 #{2} 2 #{3} 3 #{1}} 1)))))++;;; ------------------------------------------- run-dag! guard -------------------------------------------++(deftest run-dag-skips-when-already-running-test+ (testing "run-dag! does not start a second run while one is already active for the seed transform"+ (let [start-promise (promise)+ planned? (atom false)]+ (mt/with-dynamic-fn-redefs [dag-run/running-run-for-source-transform-id (constantly {:id 1})+ dag-run/start-dag-run! (fn [& _] (reset! planned? true) {:id 2})]+ (is (nil? (dag/run-dag! 42 {:direction :upstream :start-promise start-promise})))+ (is (false? @planned?) "should not create a DAG run row when one is already running")+ (is (nil? (deref start-promise 100 :timed-out))+ "start-promise is delivered nil so a waiting caller doesn't hang")))))++;; test-helpers-set-global-values!: these tests trigger an intentional duplicate-key violation,+;; which on Postgres would abort (poison) the transaction with-temp otherwise wraps the body in+(deftest start-dag-run-unique-active-run-test+ (testing "the (source_transform_id, is_active) unique constraint allows at most one active run per seed"+ (mt/test-helpers-set-global-values!+ (mt/with-temp [:model/Transform {tid :id} {:name "seed"}]+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]+ (try+ (is (thrown? Exception (dag-run/start-dag-run! tid :downstream nil 1)))+ (testing "a terminal run doesn't collide, so a new run can start"+ (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)+ (let [{run-id-2 :id} (dag-run/start-dag-run! tid :downstream nil 1)]+ (is (pos-int? run-id-2))))+ (finally+ (t2/delete! :model/TransformDagRun :source_transform_id tid))))))))++(deftest run-dag-start-race-test+ (testing "run-dag! treats losing the unique-active-run insert race as already-running"+ (mt/test-helpers-set-global-values!+ (mt/with-temp [:model/Transform {tid :id} {:name "seed" :table_dependencies []}]+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)+ start-promise (promise)]+ (try+ ;; bypass the fast pre-check to simulate two triggers racing past it together; the loser's+ ;; insert then hits the unique constraint and must be treated as "already running"+ (mt/with-dynamic-fn-redefs [dag-run/running-run-for-source-transform-id (constantly nil)]+ (is (nil? (dag/run-dag! tid {:direction :upstream :start-promise start-promise})))+ (is (nil? (deref start-promise 100 :timed-out))+ "start-promise is delivered nil so a waiting caller doesn't hang"))+ (finally+ (t2/delete! :model/TransformDagRun :id run-id))))))))++;;; ------------------------------------------- Model lifecycle -------------------------------------------++(deftest dag-run-lifecycle-test+ (mt/with-temp [:model/Transform {tid :id} {:name "seed"}]+ (testing "start-dag-run! creates a started, active run for the seed transform"+ (let [{run-id :id :as run} (dag-run/start-dag-run! tid :upstream (mt/user->id :rasta) 3)]+ (try+ (is (= :started (:status run)))+ (is (true? (:is_active run)))+ (is (= tid (:source_transform_id run)))+ (testing "the seed transform's name/entity_id are snapshotted for post-deletion display"+ (is (= "seed" (:source_transform_name run)))+ (is (some? (:source_transform_entity_id run))))+ (is (= 3 (:transform_count run)))+ (is (= :upstream (:direction run)))+ (is (= (mt/user->id :rasta) (:user_id run)))+ (testing "running-run-for-source-transform-id finds the active run"+ (is (= run-id (:id (dag-run/running-run-for-source-transform-id tid)))))+ (testing "succeed-started-run! makes it terminal and inactive"+ (is (= 1 (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)))+ (is (= :succeeded (:status (t2/select-one :model/TransformDagRun :id run-id))))+ (is (nil? (:is_active (t2/select-one :model/TransformDagRun :id run-id))))+ (is (nil? (dag-run/running-run-for-source-transform-id tid))))+ (finally+ (t2/delete! :model/TransformDagRun :id run-id)))))+ (testing "cancel-started-run! cancels an active run but is a no-op once terminal"+ (let [{run-id :id} (dag-run/start-dag-run! tid :downstream nil 1)]+ (try+ (is (= 1 (coordinated-run/cancel-started-run! :model/TransformDagRun run-id)))+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))+ (is (zero? (coordinated-run/cancel-started-run! :model/TransformDagRun run-id))+ "a finished run is never resurrected into a canceled state")+ (finally+ (t2/delete! :model/TransformDagRun :id run-id)))))+ (testing "fail-started-run! records the failure message"+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]+ (try+ (coordinated-run/fail-started-run! :model/TransformDagRun run-id {:message "boom"})+ (let [row (t2/select-one :model/TransformDagRun :id run-id)]+ (is (= :failed (:status row)))+ (is (= "boom" (:message row))))+ (finally+ (t2/delete! :model/TransformDagRun :id run-id)))))))++(deftest dag-run-survives-transform-deletion-test+ (testing "deleting the seed transform nulls source_transform_id but preserves the DAG run and its snapshot"+ (mt/with-temp [:model/Transform {tid :id} {:name "doomed seed"}]+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]+ (try+ (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)+ (t2/delete! :model/Transform :id tid)+ (let [run (t2/select-one :model/TransformDagRun :id run-id)]+ (is (some? run) "the run row survives (FK is SET NULL, not CASCADE)")+ (is (nil? (:source_transform_id run)))+ (is (= "doomed seed" (:source_transform_name run))))+ (finally+ (t2/delete! :model/TransformDagRun :id run-id)))))))++(deftest dag-run-member-runs-test+ ;; two distinct member transforms: transform_run enforces one active run per transform+ (mt/with-temp [:model/Transform {seed :id} {:name "seed"}+ :model/Transform {tid-a :id} {:name "member-a"}+ :model/Transform {tid-b :id} {:name "member-b"}]+ (let [{run-id :id} (dag-run/start-dag-run! seed :upstream nil 2)]+ (try+ (let [{active-run :id} (transform-run/start-run! tid-a {:dag_run_id run-id :run_method :manual})+ {done-run :id} (transform-run/start-run! tid-b {:dag_run_id run-id :run_method :manual})]+ (transform-run/succeed-started-run! done-run)+ (testing "transform-runs-for-dag-run returns all member runs"+ (is (= #{active-run done-run}+ (set (map :id (dag-run/transform-runs-for-dag-run run-id))))))+ (testing "cancel! cancels the run and requests cancelation of only the still-active members"+ (is (true? (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id)))+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))+ (is (t2/exists? :model/TransformRunCancelation :run_id active-run))+ (is (not (t2/exists? :model/TransformRunCancelation :run_id done-run)))+ (is (false? (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id))+ "canceling an already-terminal run is a no-op")))+ (finally+ (t2/delete! :model/TransformRun :dag_run_id run-id)+ (t2/delete! :model/TransformDagRun :id run-id))))))diff --git a/test/metabase/transforms/heartbeat_test.clj b/test/metabase/transforms/heartbeat_test.cljindex 1a1bcb89e3b..6370a7048c1 100644--- a/test/metabase/transforms/heartbeat_test.clj+++ b/test/metabase/transforms/heartbeat_test.clj@@ -6,6 +6,7 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.transforms.canceling :as canceling]+ [metabase.transforms.coordinated-run :as coordinated-run][metabase.transforms.models.job-run :as job-run][metabase.transforms.models.transform-run :as transform-run][toucan2.core :as t2])@@ -143,14 +144,14 @@:is_active nil:run_method :manual:last_heartbeat (minutes-ago 10)}]- (job-run/heartbeat-runs! [beat-id done-id])+ (coordinated-run/heartbeat-runs! :model/TransformJobRun [beat-id done-id])(testing "only the passed run that is still active gets a fresh heartbeat"(is (job-recently-beaten? beat-id))(is (not (job-recently-beaten? other-id)) "a run not in the id list is left to go stale"))(testing "an inactive run is not stamped even when passed (is_active guard)"(is (not (job-recently-beaten? done-id))))(testing "empty id list is a no-op"- (is (nil? (job-run/heartbeat-runs! [])))))))+ (is (nil? (coordinated-run/heartbeat-runs! :model/TransformJobRun [])))))))(deftest job-reap-orphaned-runs!-test(mt/with-premium-features #{:transforms-basic}diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex 8c50c564214..194ca81f8e3 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -141,9 +141,10 @@(swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)transforms.execute/execute! (fn [_ _]- (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)+ (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")(is @run-called?@@ -159,7 +160,9 @@(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message](swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)]- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) query-transform)(is (= 1 (count @logged-messages))"Should log exactly one warning")(is (= :warn (:level (first @logged-messages)))@@ -180,9 +183,10 @@(swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)transforms.execute/execute! (fn [_ _]- (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)+ (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")(is @run-called?@@ -201,9 +205,10 @@(swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)transforms.execute/execute! (fn [_ _]- (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)+ (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) query-transform)(is (= 1 (count @logged-messages))"Should log exactly one warning")(is (= :warn (:level (first @logged-messages)))@@ -236,9 +241,10 @@(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message](swap! logged conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/execute! (fn [_ _] (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! 200 :scheduled nil (promise) transform)+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job 200] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) transform)(is (false? @run-called?)"execute! must not be called when the meter is locked")(is (some #(re-matches #".*Skip running transform 7 due to locked meter.*"@@ -251,9 +257,10 @@(let [transform {:id 8 :source_type :native :source query-source :name "Unlocked"}run-called? (atom false)](mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/execute! (fn [_ _] (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! 201 :scheduled nil (promise) transform)+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job 201] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) transform)(is (true? @run-called?)))))))(testing "non-metered transform (transform-metered-as → nil) is never blocked by lock state";; Override the outer mock with one that returns nil for every source-type.@@ -264,9 +271,10 @@(let [transform {:id 9 :source_type :native :source query-source :name "Non-metered"}run-called? (atom false)](mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/execute! (fn [_ _] (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]- (#'jobs/run-transform! 202 :scheduled nil (promise) transform)+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]+ (#'jobs/run-transform! {:parent-run [:job 202] :run-method :scheduled :user-id nil+ :add-run-activity! (constantly nil)}+ (promise) transform)(is (true? @run-called?)"Non-metered transforms are not gated by lock state")))))))))@@ -514,30 +522,28 @@deps {1 #{} 2 #{1} 3 #{}}ran (atom #{})](testing "a fresh pulled-in dep is not executed, but its dependent still runs"- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- freshness/fresh-dep-ids (fn [_now dep-ids]+ (with-redefs [freshness/fresh-dep-ids (fn [_now dep-ids];; only pulled-in deps are offered for gating(is (= #{1} (set dep-ids)))#{1})- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))](is (= {::jobs/status :succeeded}- (jobs/run-transforms! 0 #{2 3} {:run-method :cron})))+ (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps} {:run-method :cron})))(is (= #{2 3} @ran) "transform 1 (fresh dep) was skipped; 2 and 3 ran")))(testing "skip-fresh-deps? false (manual run_all) force-refreshes the whole plan"(reset! ran #{})- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- freshness/fresh-dep-ids (fn [& _] (throw (ex-info "must not gate when disabled" {})))- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]+ (with-redefs [freshness/fresh-dep-ids (fn [& _] (throw (ex-info "must not gate when disabled" {})))+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))](is (= {::jobs/status :succeeded}- (jobs/run-transforms! 0 #{2 3} {:run-method :manual :skip-fresh-deps? false})))+ (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps}+ {:run-method :manual :skip-fresh-deps? false})))(is (= #{1 2 3} @ran) "with skipping disabled, the dep runs too"))))(testing "a plan that is entirely skipped exits cleanly as succeeded with nothing run"(let [ran (atom #{})]- (with-redefs [jobs/get-plan (fn [_] {:order [{:id 1}] :deps {1 #{}}})- freshness/fresh-dep-ids (fn [_now _dep-ids] #{1})- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]+ (with-redefs [freshness/fresh-dep-ids (fn [_now _dep-ids] #{1})+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))](is (= {::jobs/status :succeeded}- (jobs/run-transforms! 0 #{} {:run-method :cron})))+ (jobs/run-transforms! 0 #{} {:order [{:id 1}] :deps {1 #{}}} {:run-method :cron})))(is (= #{} @ran)))))))(deftest run-transforms!-race-condition-test@@ -594,17 +600,18 @@(catch Throwable t [nil t]))](on-exit)(if ex (throw ex) ret)))]- (let [run1 (transforms.job-run/start-run! (:id job) :manual)+ (let [plan (#'jobs/get-plan #{(:id transform)})+ run1 (transforms.job-run/start-run! (:id job) :manual)run2 (transforms.job-run/start-run! (:id job) :manual)fut1 (future(try- {:result (jobs/run-transforms! (:id run1) #{(:id transform)}+ {:result (jobs/run-transforms! (:id run1) #{(:id transform)} plan{:run-method :manual})}(catch Exception e{:error e})))fut2 (future(try- {:result (jobs/run-transforms! (:id run2) #{(:id transform)}+ {:result (jobs/run-transforms! (:id run2) #{(:id transform)} plan{:run-method :manual})}(catch Exception e{:error e})))@@ -622,33 +629,30 @@deps {1 #{} 2 #{} 3 #{} 4 #{}}barrier (CyclicBarrier. 4)](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ _]+ (with-redefs [jobs/run-transform! (fn [_ _ _](.await barrier 5 TimeUnit/SECONDS))](is (= {::jobs/status :succeeded}- (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual}))+ (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual}))"all four independents must reach the barrier (i.e. be live simultaneously)")))))(testing "Dependents wait for their dependencies even when concurrency > 1"(let [plan [{:id 1} {:id 2}]deps {1 #{} 2 #{1}}order (atom [])](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ transform]+ (with-redefs [jobs/run-transform! (fn [_ _ transform](when (= 1 (:id transform))(Thread/sleep 100))(swap! order conj (:id transform)))]- (jobs/run-transforms! 0 #{1 2} {:run-method :manual})+ (jobs/run-transforms! 0 #{1 2} {:order plan :deps deps} {:run-method :manual})(is (= [1 2] @order) "a must complete before b starts")))))(testing "Dependents of a failed transform are skipped transitively"(let [plan [{:id 1} {:id 2} {:id 3}]deps {1 #{} 2 #{1} 3 #{2}}](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ transform]+ (with-redefs [jobs/run-transform! (fn [_ _ transform](when (= 1 (:id transform))(throw (ex-info "boom" {}))))]- (let [result (jobs/run-transforms! 0 #{1 2 3} {:run-method :manual})]+ (let [result (jobs/run-transforms! 0 #{1 2 3} {:order plan :deps deps} {:run-method :manual})](is (= :failed (::jobs/status result)))(is (= #{1 2 3} (set (map (comp :id ::jobs/transform) (::jobs/failures result))))"a failed, b & c skipped as dep-failed"))))))@@ -661,13 +665,12 @@live (atom 0)max-live (atom 0)](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 2]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ _]+ (with-redefs [jobs/run-transform! (fn [_ _ _](let [n (swap! live inc)](swap! max-live max n))(.await partner-up 5 TimeUnit/SECONDS)(swap! live dec))]- (jobs/run-transforms! 0 (set (range 1 9)) {:run-method :manual})+ (jobs/run-transforms! 0 (set (range 1 9)) {:order plan :deps deps} {:run-method :manual})(is (= 2 @max-live) "should never exceed the concurrency setting")))))(testing "Python transforms are serialized in their own lane (n=1)"(let [py (fn [id] {:id id :source {:type :python}})@@ -676,15 +679,14 @@live (atom 0)max-live (atom 0)](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ _]+ (with-redefs [jobs/run-transform! (fn [_ _ _](let [n (swap! live inc)](swap! max-live max n));; A short sleep gives any (hypothetical) parallel;; python worker a window to show up in the counter.(Thread/sleep 50)(swap! live dec))]- (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual})+ (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual})(is (= 1 @max-live)"python transforms must run one at a time even with concurrency=4")))))(testing "SQL and Python lanes run in parallel without contending for slots"@@ -696,8 +698,7 @@py-live (atom 0)max-py (atom 0)](mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 3]- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})- jobs/run-transform! (fn [_ _ _ _ transform]+ (with-redefs [jobs/run-transform! (fn [_ _ transform](if (= :python (-> transform :source :type))(do (let [n (swap! py-live inc)](swap! max-py max n))@@ -705,6 +706,6 @@(swap! py-live dec))(.await sql-barrier 5 TimeUnit/SECONDS)))](is (= {::jobs/status :succeeded}- (jobs/run-transforms! 0 #{1 2 3 4 5} {:run-method :manual}))+ (jobs/run-transforms! 0 #{1 2 3 4 5} {:order plan :deps deps} {:run-method :manual}))"all three SQL transforms reach the barrier (run in parallel)")(is (= 1 @max-py) "python lane stays single-slot")))))))diff --git a/test/metabase/transforms_rest/api/transform_dag_run_test.clj b/test/metabase/transforms_rest/api/transform_dag_run_test.cljnew file mode 100644index 00000000000..196f3edc0c9--- /dev/null+++ b/test/metabase/transforms_rest/api/transform_dag_run_test.clj@@ -0,0 +1,245 @@+(ns metabase.transforms-rest.api.transform-dag-run-test+ "Tests for the /api/transform-dag-run endpoints, plus the per-transform DAG endpoints on+ /api/transform/:id (previewing and triggering a DAG-reprocess run)."+ (:require+ [clojure.test :refer :all]+ [metabase.permissions.models.permissions-group :as perms-group]+ [metabase.test :as mt]+ [metabase.transforms.test-util :refer [parse-instant utc-timestamp]]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;; transforms-enabled: transform read/write perms are gated on the transforms kill switch, which+;; defaults to off outside cloud — without it every endpoint here 403s regardless of permissions++;;; +------------------------------------------------------------------------------------------------------------------++;;; | GET /api/transform-dag-run/:run-id/transform-runs |+;;; +------------------------------------------------------------------------------------------------------------------+++(deftest get-dag-run-transform-runs-test+ (testing "GET /api/transform-dag-run/:run-id/transform-runs"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}+ :model/Transform {t1-id :id} {:name "TR1"}+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id+ :direction "upstream"+ :status "succeeded"+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/TransformRun {tr1-id :id} {:transform_id t1-id+ :dag_run_id run-id+ :status "succeeded"+ :run_method "manual"+ :transform_name "TR1"+ :start_time (parse-instant "2025-09-01T10:00:00")+ :end_time (parse-instant "2025-09-01T10:02:00")}+ :model/TransformRun {tr2-id :id} {:transform_id seed-id+ :dag_run_id run-id+ :status "failed"+ :run_method "manual"+ :transform_name "Seed"+ :start_time (parse-instant "2025-09-01T10:02:00")+ :message "Query failed"}+ ;; a run of the same transform outside the DAG run must not leak in+ :model/TransformRun _ {:transform_id t1-id+ :status "succeeded"+ :run_method "manual"+ :transform_name "TR1"+ :start_time (parse-instant "2025-08-01T10:00:00")}]+ (testing "returns the member runs, ordered by start_time ascending"+ (let [response (mt/user-http-request :lucky :get 200+ (str "transform-dag-run/" run-id "/transform-runs"))]+ (is (= [tr1-id tr2-id] (map :id response)))+ (is (every? #(= run-id (:dag_run_id %)) response))))+ (testing "response shape"+ (let [response (mt/user-http-request :lucky :get 200+ (str "transform-dag-run/" run-id "/transform-runs"))+ tr1 (first (filter #(= tr1-id (:id %)) response))]+ (is (= t1-id (:transform_id tr1)))+ (is (= "manual" (:run_method tr1)))+ (is (= "succeeded" (:status tr1)))+ (is (= "TR1" (:transform_name tr1)))+ (is (= (utc-timestamp "2025-09-01T10:00:00") (:start_time tr1)))+ (is (= (utc-timestamp "2025-09-01T10:02:00") (:end_time tr1)))))+ (testing "returns 404 for a non-existent DAG run"+ (mt/user-http-request :lucky :get 404 "transform-dag-run/999999/transform-runs"))))))))++(deftest get-dag-run-transform-runs-deleted-seed-test+ (testing "GET /api/transform-dag-run/:run-id/transform-runs still lists member runs after the seed transform is deleted"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ ;; source_transform_id nil as the SET NULL FK leaves it after the seed transform is deleted+ (mt/with-temp [:model/Transform {t1-id :id} {:name "TR1"}+ :model/TransformDagRun {run-id :id} {:source_transform_id nil+ :source_transform_name "Deleted Seed"+ :direction "upstream"+ :status "succeeded"+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/TransformRun {tr1-id :id} {:transform_id t1-id+ :dag_run_id run-id+ :status "succeeded"+ :run_method "manual"+ :transform_name "TR1"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (let [response (mt/user-http-request :lucky :get 200 (str "transform-dag-run/" run-id "/transform-runs"))]+ (is (= [tr1-id] (map :id response))))+ (testing "non-data-analysts are still denied"+ (mt/user-http-request :rasta :get 403 (str "transform-dag-run/" run-id "/transform-runs")))))))))++(deftest get-dag-run-transform-runs-requires-read-permission-test+ (testing "GET /api/transform-dag-run/:run-id/transform-runs requires read permission on the seed transform"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id+ :direction "upstream"+ :status "succeeded"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :rasta :get 403 (str "transform-dag-run/" run-id "/transform-runs")))))))++;;; +------------------------------------------------------------------------------------------------------------------++;;; | POST /api/transform-dag-run/:run-id/cancel |+;;; +------------------------------------------------------------------------------------------------------------------+++(deftest cancel-dag-run-test+ (testing "POST /api/transform-dag-run/:run-id/cancel"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ ;; canceling write-checks the seed transform, which needs the transforms permission on its source DB+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (testing "cancels an active DAG run and requests cancellation of its active members"+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}+ :model/Transform {t1-id :id} {:name "TR1"}+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id+ :direction "upstream"+ :status "started"+ :is_active true+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/TransformRun {member-id :id} {:transform_id t1-id+ :dag_run_id run-id+ :status "started"+ :is_active true+ :run_method "manual"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :lucky :post 204 (str "transform-dag-run/" run-id "/cancel"))+ (let [run (t2/select-one :model/TransformDagRun :id run-id)]+ (is (= :canceled (:status run)))+ (is (nil? (:is_active run)))+ (is (some? (:end_time run))))+ (testing "a cancelation row is recorded for the still-running member"+ (is (t2/exists? :model/TransformRunCancelation :run_id member-id)))))+ (testing "returns 400 when the run has already finished"+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id+ :direction "downstream"+ :status "succeeded"+ :start_time (parse-instant "2025-09-01T10:00:00")+ :end_time (parse-instant "2025-09-01T10:05:00")}]+ (mt/user-http-request :lucky :post 400 (str "transform-dag-run/" run-id "/cancel"))+ (is (= :succeeded (:status (t2/select-one :model/TransformDagRun :id run-id)))+ "a finished run is never resurrected into a canceled state")))+ (testing "returns 404 for a non-existent DAG run"+ (mt/user-http-request :lucky :post 404 "transform-dag-run/999999/cancel"))))))))++(deftest cancel-dag-run-deleted-seed-test+ (testing "POST /api/transform-dag-run/:run-id/cancel still works after the seed transform is deleted"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/TransformDagRun {run-id :id} {:source_transform_id nil+ :source_transform_name "Deleted Seed"+ :direction "upstream"+ :status "started"+ :is_active true+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :lucky :post 204 (str "transform-dag-run/" run-id "/cancel"))+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))))))))++(deftest cancel-dag-run-requires-write-permission-test+ (testing "POST /api/transform-dag-run/:run-id/cancel requires write permission on the seed transform"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id+ :direction "upstream"+ :status "started"+ :is_active true+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :rasta :post 403 (str "transform-dag-run/" run-id "/cancel")))))))++;;; +------------------------------------------------------------------------------------------------------------------++;;; | GET /api/transform/:id/dag-transforms |+;;; +------------------------------------------------------------------------------------------------------------------+++(deftest dag-transforms-preview-test+ (testing "GET /api/transform/:id/dag-transforms"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ ;; consumer reads producer's output table via the stored table_dependencies column, so the+ ;; dependency graph resolves without parsing any queries+ (mt/with-temp [:model/Transform {producer-id :id} {:name "Producer"+ :target_table_id (mt/id :orders)+ :table_dependencies []}+ :model/Transform {consumer-id :id} {:name "Consumer"+ :table_dependencies [{:table (mt/id :orders)}]}]+ (testing "upstream preview returns the dependency closure in execution order"+ (is (= [{:id producer-id :name "Producer"}+ {:id consumer-id :name "Consumer"}]+ (mt/user-http-request :lucky :get 200 (str "transform/" consumer-id "/dag-transforms")+ :direction "upstream"))))+ (testing "upstream preview of a transform with no dependencies is just itself"+ (is (= [{:id producer-id :name "Producer"}]+ (mt/user-http-request :lucky :get 200 (str "transform/" producer-id "/dag-transforms")+ :direction "upstream"))))+ (testing "downstream preview returns the dependents closure in execution order"+ (is (= [{:id producer-id :name "Producer"}+ {:id consumer-id :name "Consumer"}]+ (mt/user-http-request :lucky :get 200 (str "transform/" producer-id "/dag-transforms")+ :direction "downstream"))))+ (testing "direction is required and validated"+ (mt/user-http-request :lucky :get 400 (str "transform/" consumer-id "/dag-transforms"))+ (mt/user-http-request :lucky :get 400 (str "transform/" consumer-id "/dag-transforms")+ :direction "sideways"))+ (testing "returns 404 for a non-existent transform"+ (mt/user-http-request :lucky :get 404 "transform/999999/dag-transforms"+ :direction "upstream"))))))))++;;; +------------------------------------------------------------------------------------------------------------------++;;; | POST /api/transform/:id/run-dag |+;;; +------------------------------------------------------------------------------------------------------------------+++(deftest run-dag-already-running-test+ (testing "POST /api/transform/:id/run-dag returns 202 with a nil dag_run_id when a DAG run is already in progress"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}+ :model/TransformDagRun _ {:source_transform_id tid+ :direction "upstream"+ :status "started"+ :is_active true+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (let [response (mt/user-http-request :crowberto :post 202 (str "transform/" tid "/run-dag")+ {:direction "upstream"})]+ (is (nil? (:dag_run_id response)))))))))++(deftest run-dag-validates-direction-test+ (testing "POST /api/transform/:id/run-dag requires a valid direction"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}]+ (mt/user-http-request :crowberto :post 400 (str "transform/" tid "/run-dag") {})+ (mt/user-http-request :crowberto :post 400 (str "transform/" tid "/run-dag")+ {:direction "sideways"}))))))++(deftest run-dag-requires-write-permission-test+ (testing "POST /api/transform/:id/run-dag requires write permission on the transform"+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}]+ (mt/user-http-request :rasta :post 403 (str "transform/" tid "/run-dag")+ {:direction "upstream"}))))))diff --git a/test/metabase/transforms_rest/api/transform_job_test.clj b/test/metabase/transforms_rest/api/transform_job_test.cljindex ea96ed1f71f..9c510118537 100644--- a/test/metabase/transforms_rest/api/transform_job_test.clj+++ b/test/metabase/transforms_rest/api/transform_job_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase.test :as mt]+ [metabase.transforms.jobs :as jobs][metabase.transforms.models.transform-job][metabase.transforms.models.transform-run][metabase.transforms.models.transform-tag]@@ -308,15 +309,42 @@(mt/user-http-request :lucky :delete 404 "transform-job/999999")))))))(deftest execute-job-test- (testing "POST /api/transform-job/:id/execute"+ (testing "POST /api/transform-job/:id/run"(mt/with-data-analyst-role! (mt/user->id :lucky)(mt/with-premium-features #{:transforms-basic}- (mt/with-temp [:model/TransformJob job {:name "To Execute" :schedule "0 0 0 * * ?"}]- (testing "Returns stub run response"- (let [response (mt/user-http-request :lucky :post 200 (str "transform-job/" (:id job) "/run"))]- (is (= "Job run started" (:message response)))- (is (string? (:job_run_id response)))- (is (re-matches #"stub-\d+-\d+" (:job_run_id response))))))))))+ (testing "returns the created run id"+ (mt/with-model-cleanup [:model/TransformJobRun]+ (mt/with-temp [:model/TransformTag tag {:name "execute-test-tag"}+ :model/TransformJob job {:name "To Execute" :schedule "0 0 0 * * ?"}+ :model/TransformJobTransformTag _ {:job_id (:id job) :tag_id (:id tag) :position 0}+ :model/Transform transform {:name "To Run"}+ :model/TransformTransformTag _ {:transform_id (:id transform) :tag_id (:id tag) :position 0}]+ (let [called (promise)]+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]+ (deliver called run-id)+ {::jobs/status :succeeded})]+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]+ (is (= "Job run started" (:message response)))+ (is (pos-int? (:job_run_id response)))+ (is (= (:job_run_id response) (deref called 5000 ::timed-out)))+ (let [run (t2/select-one :model/TransformJobRun :id (:job_run_id response))]+ (is (= (:id job) (:job_id run)))+ (testing "the job's name and entity_id are snapshotted on the run"+ (is (= "To Execute" (:job_name run)))+ (is (= (:entity_id job) (:job_entity_id run)))))))))))+ (testing "returns a nil run id when the job has no transforms"+ (mt/with-temp [:model/TransformJob job {:name "Empty Job" :schedule "0 0 0 * * ?"}]+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]+ (is (nil? (:job_run_id response))))))+ (testing "returns a nil run id when the job is already running"+ (mt/with-temp [:model/TransformJob job {:name "Busy Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun _ {:job_id (:id job)+ :status "started"+ :is_active true+ :run_method "manual"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]+ (is (nil? (:job_run_id response))))))))))(deftest permissions-test(testing "All endpoints require transform permissions"@@ -507,6 +535,70 @@(mt/user-http-request :rasta :get 403(str "transform-job/" job-id "/runs/" run-id "/transform-runs"))))))+;;; +------------------------------------------------------------------------------------------------------------------++;;; | POST /api/transform-job/:job-id/runs/:run-id/cancel |+;;; +------------------------------------------------------------------------------------------------------------------+++(deftest cancel-job-run-test+ (testing "POST /api/transform-job/:job-id/runs/:run-id/cancel"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (testing "cancels an active job run and requests cancellation of its active members"+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {run-id :id} {:job_id job-id+ :status "started"+ :is_active true+ :run_method "manual"+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/Transform {t1-id :id} {:name "TR1"}+ :model/TransformRun {member-id :id} {:transform_id t1-id+ :job_run_id run-id+ :status "started"+ :is_active true+ :run_method "cron"+ :start_time (parse-instant "2025-09-01T10:00:00")+ :end_time nil}]+ (mt/user-http-request :lucky :post 204 (str "transform-job/" job-id "/runs/" run-id "/cancel"))+ (let [run (t2/select-one :model/TransformJobRun :id run-id)]+ (is (= :canceled (:status run)))+ (is (nil? (:is_active run)))+ (is (some? (:end_time run))))+ (testing "a cancelation row is recorded for the still-running member"+ (is (t2/exists? :model/TransformRunCancelation :run_id member-id)))))+ (testing "returns 400 when the run has already finished"+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {run-id :id} {:job_id job-id+ :status "succeeded"+ :run_method "cron"+ :start_time (parse-instant "2025-09-01T10:00:00")+ :end_time (parse-instant "2025-09-01T10:05:00")}]+ (mt/user-http-request :lucky :post 400 (str "transform-job/" job-id "/runs/" run-id "/cancel"))+ (is (= :succeeded (:status (t2/select-one :model/TransformJobRun :id run-id)))+ "a finished run is never resurrected into a canceled state")))+ (testing "returns 404 for a non-existent job or run, or a run of another job"+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}+ :model/TransformJob {other-job-id :id} {:name "Other Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {run-id :id} {:job_id job-id+ :status "started"+ :is_active true+ :run_method "manual"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :lucky :post 404 (str "transform-job/999999/runs/" run-id "/cancel"))+ (mt/user-http-request :lucky :post 404 (str "transform-job/" job-id "/runs/999999/cancel"))+ (mt/user-http-request :lucky :post 404 (str "transform-job/" other-job-id "/runs/" run-id "/cancel"))))))))++(deftest cancel-job-run-requires-write-permission-test+ (testing "POST /api/transform-job/:job-id/runs/:run-id/cancel requires write permission on the job"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Auth Test" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {run-id :id} {:job_id job-id+ :status "started"+ :is_active true+ :run_method "manual"+ :start_time (parse-instant "2025-09-01T10:00:00")}]+ (mt/user-http-request :rasta :post 403+ (str "transform-job/" job-id "/runs/" run-id "/cancel"))))))+(deftest built-in-jobs-and-tags-seeded-test(testing "the built-in Hourly/Daily/Weekly/Monthly jobs and tags are pre-seeded (migration data)";; Looked up by :name (stable) rather than :entity_id/:built_in_type: editing a built-in job indiff --git a/test/metabase/transforms_rest/api/transform_runs_test.clj b/test/metabase/transforms_rest/api/transform_runs_test.cljnew file mode 100644index 00000000000..34d63a99754--- /dev/null+++ b/test/metabase/transforms_rest/api/transform_runs_test.clj@@ -0,0 +1,234 @@+(ns metabase.transforms-rest.api.transform-runs-test+ "Tests for GET /api/transform/runs — the unified listing of root runs (job runs, manual+ DAG-reprocess runs, and standalone transform runs)."+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.transforms.test-util :refer [parse-instant utc-timestamp]]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- rows-by-type+ "Index the response rows by `[run_type id]` so assertions are immune to rows created by other+ tests (the listing is global)."+ [response]+ (into {} (map (juxt (juxt :run_type :id) identity)) (:data response)))++(defn- do-with-runs-fixture!+ "Call `f` with a map of fixture ids: a job run whose member ran `transform-a`, a DAG run seeded+ from (and whose member ran) `transform-b`, and a standalone run of `transform-a`."+ [f]+ (mt/with-temp [:model/Transform {ta-id :id} {:name "Transform A"}+ :model/Transform {tb-id :id} {:name "Transform B"}+ :model/TransformJob {job-id :id} {:name "Nightly Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {job-run-id :id} {:job_id job-id+ :status "succeeded"+ :run_method "cron"+ :start_time (parse-instant "2025-09-01T10:00:00")+ :end_time (parse-instant "2025-09-01T10:05:00")}+ :model/TransformRun {member-a-id :id} {:transform_id ta-id+ :job_run_id job-run-id+ :status "succeeded"+ :run_method "cron"+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/TransformDagRun {dag-run-id :id} {:source_transform_id tb-id+ :direction "downstream"+ :transform_count 4+ :status "failed"+ :message "boom"+ :user_id (mt/user->id :lucky)+ :start_time (parse-instant "2025-09-02T10:00:00")+ :end_time (parse-instant "2025-09-02T10:03:00")}+ :model/TransformRun {member-b-id :id} {:transform_id tb-id+ :dag_run_id dag-run-id+ :status "failed"+ :run_method "manual"+ :start_time (parse-instant "2025-09-02T10:00:00")}+ :model/TransformRun {standalone-id :id} {:transform_id ta-id+ :status "succeeded"+ :run_method "manual"+ :user_id (mt/user->id :lucky)+ :start_time (parse-instant "2025-09-03T10:00:00")+ :end_time (parse-instant "2025-09-03T10:01:00")}]+ (f {:ta-id ta-id, :tb-id tb-id, :job-id job-id, :job-run-id job-run-id, :dag-run-id dag-run-id+ :member-a-id member-a-id, :member-b-id member-b-id, :standalone-id standalone-id})))++(defmacro ^:private with-runs-fixture!+ "Execute `body` with `binding` bound to the fixture-id map of [[do-with-runs-fixture!]]."+ [[binding] & body]+ `(do-with-runs-fixture! (fn [~binding] ~@body)))++(defn- get-runs [& params]+ ;; a large limit so fixture rows can't be pushed off the page by rows other tests created+ (apply mt/user-http-request :lucky :get 200 "transform/runs" :limit 1000 params))++(deftest unified-runs-listing-test+ (testing "GET /api/transform/runs"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (with-runs-fixture! [{:keys [ta-id tb-id job-id job-run-id dag-run-id member-a-id member-b-id standalone-id]}]+ (let [response (get-runs)+ rows (rows-by-type response)]+ (testing "returns all three kinds of root run"+ (is (contains? rows ["job" job-run-id]))+ (is (contains? rows ["dag" dag-run-id]))+ (is (contains? rows ["transform" standalone-id])))+ (testing "member runs of a job/DAG run are not listed as rows"+ (is (not (contains? rows ["transform" member-a-id])))+ (is (not (contains? rows ["transform" member-b-id]))))+ (testing "pagination envelope"+ (is (<= 3 (:total response)))+ (is (pos? (:limit response)))+ (is (integer? (:offset response))))+ (testing "job row shape"+ (let [row (rows ["job" job-run-id])]+ (is (= job-id (:entity_id row)))+ (is (= "Nightly Job" (:name row)))+ (is (nil? (:direction row)))+ (is (nil? (:transform_count row)))+ (is (= "cron" (:run_method row)))+ (is (= "succeeded" (:status row)))+ (is (= (utc-timestamp "2025-09-01T10:00:00") (:start_time row)))+ (is (= (utc-timestamp "2025-09-01T10:05:00") (:end_time row)))))+ (testing "dag row shape"+ (let [row (rows ["dag" dag-run-id])]+ (is (= tb-id (:entity_id row)))+ (is (= "Transform B" (:name row)))+ (is (= "downstream" (:direction row)))+ (is (= 4 (:transform_count row)))+ (is (= "manual" (:run_method row)))+ (is (= "failed" (:status row)))+ (is (= "boom" (:message row)))+ (is (= (mt/user->id :lucky) (:user_id row)))))+ (testing "standalone transform row shape"+ (let [row (rows ["transform" standalone-id])]+ (is (= ta-id (:entity_id row)))+ (is (= "Transform A" (:name row)))+ (is (nil? (:direction row)))+ (is (= "manual" (:run_method row)))+ (is (= (mt/user->id :lucky) (:user_id row)))))))))))++(deftest unified-runs-filters-test+ (testing "GET /api/transform/runs filters"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (with-runs-fixture! [{:keys [ta-id tb-id job-run-id dag-run-id standalone-id]}]+ (testing "types= restricts the kinds returned"+ (let [response (get-runs :types ["job"])]+ (is (every? #(= "job" (:run_type %)) (:data response)))+ (is (contains? (rows-by-type response) ["job" job-run-id])))+ (let [response (get-runs :types ["dag" "transform"])+ rows (rows-by-type response)]+ (is (every? #(contains? #{"dag" "transform"} (:run_type %)) (:data response)))+ (is (contains? rows ["dag" dag-run-id]))+ (is (contains? rows ["transform" standalone-id]))))+ (testing "statuses= filters across all kinds, matching any of the given statuses"+ (let [rows (rows-by-type (get-runs :statuses ["failed"]))]+ (is (contains? rows ["dag" dag-run-id]))+ (is (not (contains? rows ["job" job-run-id])))+ (is (not (contains? rows ["transform" standalone-id]))))+ (let [rows (rows-by-type (get-runs :statuses ["failed" "succeeded"]))]+ (is (contains? rows ["dag" dag-run-id]))+ (is (contains? rows ["job" job-run-id]))+ (is (contains? rows ["transform" standalone-id]))))+ (testing "run-methods= filters by trigger"+ (let [rows (rows-by-type (get-runs :run-methods ["cron"]))]+ (is (contains? rows ["job" job-run-id]))+ (is (not (contains? rows ["dag" dag-run-id])))+ (is (not (contains? rows ["transform" standalone-id]))))+ (let [rows (rows-by-type (get-runs :run-methods ["manual"]))]+ (is (contains? rows ["dag" dag-run-id]))+ (is (contains? rows ["transform" standalone-id]))+ (is (not (contains? rows ["job" job-run-id])))))+ (testing "transform-ids= returns runs that ran any of the transforms"+ (testing "job runs with a member run of it, plus its standalone runs"+ (let [rows (rows-by-type (get-runs :transform-ids [ta-id]))]+ (is (contains? rows ["job" job-run-id]))+ (is (contains? rows ["transform" standalone-id]))+ (is (not (contains? rows ["dag" dag-run-id])))))+ (testing "DAG runs with a member run of it"+ (let [rows (rows-by-type (get-runs :transform-ids [tb-id]))]+ (is (contains? rows ["dag" dag-run-id]))+ (is (not (contains? rows ["job" job-run-id])))+ (is (not (contains? rows ["transform" standalone-id])))))+ (testing "multiple ids match as a logical OR"+ (let [rows (rows-by-type (get-runs :transform-ids [ta-id tb-id]))]+ (is (contains? rows ["job" job-run-id]))+ (is (contains? rows ["dag" dag-run-id]))+ (is (contains? rows ["transform" standalone-id])))))+ (testing "start-time= filters by run start"+ (let [rows (rows-by-type (get-runs :start-time "2025-09-03"))]+ (is (contains? rows ["transform" standalone-id]))+ (is (not (contains? rows ["job" job-run-id])))+ (is (not (contains? rows ["dag" dag-run-id])))))+ (testing "end-time= filters by run end"+ (let [rows (rows-by-type (get-runs :end-time "2025-09-01"))]+ (is (contains? rows ["job" job-run-id]))+ (is (not (contains? rows ["dag" dag-run-id])))+ (is (not (contains? rows ["transform" standalone-id]))))))))))++(deftest unified-runs-sorting-and-pagination-test+ (testing "GET /api/transform/runs sorting and pagination"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (with-runs-fixture! [{:keys [job-run-id dag-run-id standalone-id]}]+ (testing "sort-column=start_time asc keeps our rows in chronological order"+ (let [response (get-runs :sort-column "start_time" :sort-direction "asc")+ ours (filter (comp #{["job" job-run-id] ["dag" dag-run-id] ["transform" standalone-id]}+ (juxt :run_type :id))+ (:data response))]+ ;; rows from other tests may interleave; the relative order of ours must hold+ (is (= [["job" job-run-id] ["dag" dag-run-id] ["transform" standalone-id]]+ (map (juxt :run_type :id) ours)))))+ (testing "limit/offset are honored"+ (let [response (mt/user-http-request :lucky :get 200 "transform/runs" :limit 1 :offset 0)]+ (is (= 1 (count (:data response))))+ (is (= 1 (:limit response)))+ (is (= 0 (:offset response)))+ (is (<= 3 (:total response))))))))))++(deftest unified-runs-deleted-entities-test+ (testing "GET /api/transform/runs keeps rows for deleted jobs/transforms, naming them from the run-start snapshot"+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Transform {ta-id :id} {:name "Doomed Transform"}+ :model/Transform {tb-id :id} {:name "Doomed Seed"}+ :model/TransformJob {job-id :id} {:name "Doomed Job" :schedule "0 0 0 * * ?"}+ :model/TransformJobRun {job-run-id :id} {:job_id job-id+ :job_name "Doomed Job"+ :status "succeeded"+ :run_method "cron"+ :start_time (parse-instant "2025-09-01T10:00:00")}+ :model/TransformDagRun {dag-run-id :id} {:source_transform_id tb-id+ :source_transform_name "Doomed Seed"+ :direction "upstream"+ :status "succeeded"+ :start_time (parse-instant "2025-09-02T10:00:00")}+ :model/TransformRun {standalone-id :id} {:transform_id ta-id+ :transform_name "Doomed Transform"+ :status "succeeded"+ :run_method "manual"+ :start_time (parse-instant "2025-09-03T10:00:00")}]+ (t2/delete! :model/TransformJob :id job-id)+ (t2/delete! :model/Transform :id [:in [ta-id tb-id]])+ (let [rows (rows-by-type (get-runs))]+ (testing "job run survives (job_id has no FK) with its snapshot name"+ (let [row (rows ["job" job-run-id])]+ (is (some? row))+ (is (= "Doomed Job" (:name row)))))+ (testing "DAG run survives (FK is SET NULL) with a nil entity_id and its snapshot name"+ (let [row (rows ["dag" dag-run-id])]+ (is (some? row))+ (is (nil? (:entity_id row)))+ (is (= "Doomed Seed" (:name row)))))+ (testing "standalone run survives (FK is SET NULL) with a nil entity_id and its snapshot name"+ (let [row (rows ["transform" standalone-id])]+ (is (some? row))+ (is (nil? (:entity_id row)))+ (is (= "Doomed Transform" (:name row)))))))))))++(deftest unified-runs-requires-data-analyst-test+ (testing "GET /api/transform/runs requires the data-analyst role"+ (mt/with-premium-features #{:transforms-basic}+ (mt/user-http-request :rasta :get 403 "transform/runs"))))