Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Transform run dag be (#76974)

Nicola Mometto30021f1abd12
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 216b43e77e2..bc4ebdbcb70 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -526,6 +526,7 @@
metabase.test/with-temp-empty-app-db clojure.core/let
metabase.test/with-temp-file clojure.core/let
metabase.test/with-user-in-groups clojure.core/let
+ metabase.transforms-rest.api.transform-runs-test/with-runs-fixture! clojure.core/fn
metabase.upload.impl-test/with-upload-table! clojure.core/let
metabase.util/for-map clojure.core/for
metabase.util.files/with-open-path-to-resource clojure.core/let
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 02f0f0d4340..19d4a54f271 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2447,6 +2447,7 @@
:friends #{models
enterprise/transforms-python}
:model-exports #{:model/Transform
+ :model/TransformDagRun
:model/TransformJob
:model/TransformJobRun
:model/TransformJobTransformTag
@@ -2504,6 +2505,7 @@
transforms-base
util}
:model-imports #{:model/Transform
+ :model/TransformDagRun
:model/TransformJob
:model/TransformJobRun
:model/TransformJobTransformTag
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index 62c9b93f793..7629b344b3d 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -1919,7 +1919,7 @@ LIMIT
getRunStatus().should("have.text", "Run in progress…");
getCancelButton().click();
- H.modal().button("Yes").click();
+ H.modal().button("Cancel run").click();
getRunButton().should("have.text", "Canceling…");
getRunStatus().should("have.text", "Canceling…");
@@ -1938,6 +1938,7 @@ LIMIT
getRunStatus().should("have.text", "Run in progress…");
getRunsNavLink().click();
+ getDetailedViewSwitch().click();
getTransformRunTable().findByText("In progress").click();
cy.findByTestId("run-list-sidebar").button("Cancel run").click();
H.modal().button("Yes").click();
@@ -1956,7 +1957,7 @@ LIMIT
getRunStatus().should("have.text", "Run in progress…");
getCancelButton().click();
- H.modal().button("Yes").click();
+ H.modal().button("Cancel run").click();
getRunButton().should("have.text", "Canceling…");
getRunStatus().should("have.text", "Canceling…");
@@ -3110,6 +3111,7 @@ describe("scenarios > admin > transforms > jobs", () => {
.should("be.visible");
getRunsNavLink().click();
+ getDetailedViewSwitch().click();
getTransformRunTable().within(() => {
cy.findByText("MBQL transform").should("be.visible");
cy.findByText("Success").should("be.visible");
@@ -3651,6 +3653,7 @@ describe("scenarios > admin > transforms > runs", () => {
createInitialData();
getRunsNavLink().click();
+ getDetailedViewSwitch().click();
testTransformFilter();
testStatusFilter();
testTagFilter();
@@ -3700,6 +3703,7 @@ describe("scenarios > admin > transforms > runs", () => {
createInitialData();
getRunsNavLink().click();
+ getDetailedViewSwitch().click();
// ascending: "MBQL transform" < "SQL transform"
testSorting({
@@ -4033,7 +4037,11 @@ function openBulkActionsMenu() {
}
function visitRunListPage() {
- return cy.visit("/data-studio/transforms/runs");
+ return cy.visit("/data-studio/transforms/runs/individual");
+}
+
+function getDetailedViewSwitch() {
+ return cy.findByTestId("detailed-view-switch");
}
function runTransformAndWaitForSuccess() {
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index b941d926289..7dfe45dbdc4 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -153,6 +153,7 @@
"TaskHistory"
"TaskRun"
"Tenant"
+ "TransformDagRun"
"TransformJobRun"
"TransformRun"
"TransformRunCancelation"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 8315836fff5..e2ace811569 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -97,7 +97,7 @@
"Execute a Python transform by calling the python runner.
Blocks until the transform returns."
- [transform {:keys [run-method on-start user-id job-run-id]}]
+ [transform {:keys [run-method on-start user-id parent-run]}]
(try
(let [message-log (base/empty-message-log)
{:keys [target owner_user_id creator_id] transform-id :id} transform
@@ -105,7 +105,8 @@
run-user-id (if (and (= run-method :manual) user-id)
user-id
(or owner_user_id creator_id))
- {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id :job-run-id job-run-id)]
+ {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id
+ :parent-run parent-run)]
(when on-start (on-start run-id))
(with-open [_ (open-python-message-update-future! run-id message-log)]
(driver.conn/with-write-connection
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 38bb58f3360..b70654cffc9 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -134,6 +134,7 @@
;; TODO we should remove these models from here once serialization is supported
:model/TransformRun
:model/TransformRunCancelation
+ :model/TransformDagRun
:model/TransformJobRun
:model/TransformJobTransformTag
:model/TransformTransformTag
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
index 0e42e850901..1f013bc5733 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -62,7 +62,12 @@
(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)]
- (#'jobs/run-transform! run-id :scheduled nil (promise) python-transform)
+ (#'jobs/run-transform! {:parent-run [:job run-id]
+ :run-method :scheduled
+ :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise)
+ python-transform)
(is (= 1 (count @logged-messages))
"Should log exactly one warning")
(is (= :warn (:level (first @logged-messages)))
diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.ts
index 47e34fd89c7..c02f5eb3ef9 100644
--- a/frontend/src/metabase-types/api/mocks/transform.ts
+++ b/frontend/src/metabase-types/api/mocks/transform.ts
@@ -2,10 +2,12 @@ import type {
DatabaseId,
InspectorCard,
InspectorSource,
+ ListTransformGraphRunsResponse,
ListTransformJobRunsResponse,
ListTransformRunsResponse,
PythonTransformTableAliases,
Transform,
+ TransformGraphRun,
TransformJob,
TransformJobRun,
TransformOwner,
@@ -185,6 +187,39 @@ export function createMockTransformRunForJobRun(
};
}
+export function createMockTransformGraphRun(
+ opts?: Partial<TransformGraphRun>,
+): TransformGraphRun {
+ return {
+ run_type: "job",
+ id: 1,
+ entity_id: 1,
+ name: "Graph run",
+ direction: null,
+ transform_count: null,
+ run_method: "manual",
+ status: "succeeded",
+ is_active: false,
+ start_time: "2024-01-01T00:00:00Z",
+ end_time: "2024-01-01T00:01:00Z",
+ message: null,
+ user_id: null,
+ ...opts,
+ };
+}
+
+export function createMockListTransformGraphRunsResponse(
+ opts?: Partial<ListTransformGraphRunsResponse>,
+): ListTransformGraphRunsResponse {
+ return {
+ data: [],
+ total: 0,
+ limit: null,
+ offset: null,
+ ...opts,
+ };
+}
+
export function createMockTransformTag(
opts?: Partial<TransformTag>,
): TransformTag {
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 99c055c9a02..6fb5dfab6dd 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -379,6 +379,79 @@ export type ListJobRunTransformRunsRequest = {
runId: TransformJobRunId;
};
+export type CancelJobRunRequest = {
+ jobId: TransformJobId;
+ runId: TransformJobRunId;
+};
+
+export type TransformDagRunId = number;
+
+export const TRANSFORM_DAG_DIRECTIONS = ["upstream", "downstream"] as const;
+export type TransformDagDirection = (typeof TRANSFORM_DAG_DIRECTIONS)[number];
+
+export type RunTransformDagRequest = {
+ id: TransformId;
+ direction: TransformDagDirection;
+ skip_fresh_deps?: boolean;
+};
+
+export type RunTransformDagResponse = {
+ dag_run_id: TransformDagRunId | null;
+ message: string;
+};
+
+export type ListDagTransformsRequest = {
+ transformId: TransformId;
+ direction: TransformDagDirection;
+};
+
+export type DagTransform = Pick<Transform, "id" | "name">;
+
+export type ListDagRunTransformRunsRequest = {
+ dagRunId: TransformDagRunId;
+};
+
+export const TRANSFORM_GRAPH_RUN_TYPES = ["job", "dag", "transform"] as const;
+export type TransformGraphRunType = (typeof TRANSFORM_GRAPH_RUN_TYPES)[number];
+
+export const TRANSFORM_GRAPH_RUN_SORT_COLUMNS = [
+ "start_time",
+ "end_time",
+] as const;
+export type TransformGraphRunSortColumn =
+ (typeof TRANSFORM_GRAPH_RUN_SORT_COLUMNS)[number];
+
+export type TransformGraphRun = {
+ run_type: TransformGraphRunType;
+ id: number;
+ entity_id: number | null;
+ name: string | null;
+ direction: TransformDagDirection | null;
+ transform_count: number | null;
+ run_method: TransformRunMethod | null;
+ status: TransformRunStatus | null;
+ is_active: boolean | null;
+ start_time: string;
+ end_time: string | null;
+ message: string | null;
+ user_id: UserId | null;
+};
+
+export type ListTransformGraphRunsRequest = {
+ types?: TransformGraphRunType[];
+ statuses?: TransformRunStatus[];
+ "transform-ids"?: TransformId[];
+ "start-time"?: string;
+ "end-time"?: string;
+ "run-methods"?: TransformRunMethod[];
+ "sort-column"?: TransformGraphRunSortColumn;
+ "sort-direction"?: SortDirection;
+} & PaginationRequest;
+
+export type ListTransformGraphRunsResponse = {
+ data: TransformGraphRun[];
+} & PaginationResponse;
+
export type TestPythonTransformRequest = {
code: string;
source_tables: PythonTransformTableAliases;
diff --git a/frontend/src/metabase/api/tags/constants.ts b/frontend/src/metabase/api/tags/constants.ts
index 25b9167f7d1..a5ba21c986b 100644
--- a/frontend/src/metabase/api/tags/constants.ts
+++ b/frontend/src/metabase/api/tags/constants.ts
@@ -61,6 +61,7 @@ export const TAG_TYPES = [
"transform-job",
"transform-job-via-tag",
"transform-run",
+ "transform-dag-run",
"transform-inspector-lens",
"user",
"public-dashboard",
diff --git a/frontend/src/metabase/api/transform-job.ts b/frontend/src/metabase/api/transform-job.ts
index 1164e5c01f6..67cf0828e43 100644
--- a/frontend/src/metabase/api/transform-job.ts
+++ b/frontend/src/metabase/api/transform-job.ts
@@ -82,6 +82,8 @@ export const transformJobApi = Api.injectEndpoints({
invalidatesTags: (_, error, id) =>
invalidateTags(error, [
idTag("transform-job", id),
+ listTag("transform-job"),
+ listTag("transform-run"),
tag("transform"),
tag("table"),
listTag("table-remapping"),
diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts
index 266c21b491c..3690054d8d1 100644
--- a/frontend/src/metabase/api/transform.ts
+++ b/frontend/src/metabase/api/transform.ts
@@ -1,17 +1,27 @@
import { isResourceNotFoundError } from "metabase/utils/errors";
import type {
+ CancelJobRunRequest,
CreateTransformRequest,
+ DagTransform,
Dataset,
GetInspectorLensRequest,
InspectorDiscoveryResponse,
InspectorLens,
+ ListDagRunTransformRunsRequest,
+ ListDagTransformsRequest,
+ ListTransformGraphRunsRequest,
+ ListTransformGraphRunsResponse,
ListTransformRunsRequest,
ListTransformRunsResponse,
ListTransformsRequest,
RunInspectorQueryRequest,
+ RunTransformDagRequest,
+ RunTransformDagResponse,
RunTransformResponse,
Transform,
+ TransformDagRunId,
TransformId,
+ TransformRunForJobRun,
UpdateTransformRequest,
} from "metabase-types/api";
@@ -106,7 +116,11 @@ export const transformApi = Api.injectEndpoints({
url: `/api/transform/${id}/cancel`,
}),
invalidatesTags: (_, error, id) =>
- invalidateTags(error, [idTag("transform", id), tag("table")]),
+ invalidateTags(error, [
+ idTag("transform", id),
+ tag("table"),
+ listTag("transform-run"),
+ ]),
onQueryStarted: async (id, { dispatch, queryFulfilled }) => {
const patchResult = dispatch(
transformApi.util.updateQueryData("getTransform", id, (draft) => {
@@ -139,6 +153,88 @@ export const transformApi = Api.injectEndpoints({
}
},
}),
+ runTransformDag: builder.mutation<
+ RunTransformDagResponse,
+ RunTransformDagRequest
+ >({
+ query: ({ id, ...body }) => ({
+ method: "POST",
+ url: `/api/transform/${id}/run-dag`,
+ body,
+ }),
+ invalidatesTags: (_, error, { id }) =>
+ invalidateTags(error, [
+ idTag("transform", id),
+ tag("transform"),
+ tag("table"),
+ listTag("table-remapping"),
+ listTag("transform-run"),
+ listTag("transform-dag-run"),
+ ]),
+ }),
+ listDagTransforms: builder.query<DagTransform[], ListDagTransformsRequest>({
+ query: ({ transformId, direction }) => ({
+ method: "GET",
+ url: `/api/transform/${transformId}/dag-transforms`,
+ params: { direction },
+ }),
+ providesTags: (_response, _error, { transformId }) => [
+ idTag("transform", transformId),
+ ],
+ }),
+ listDagRunTransformRuns: builder.query<
+ TransformRunForJobRun[],
+ ListDagRunTransformRunsRequest
+ >({
+ query: ({ dagRunId }) => ({
+ method: "GET",
+ url: `/api/transform-dag-run/${dagRunId}/transform-runs`,
+ }),
+ providesTags: (_response, _error, { dagRunId }) => [
+ idTag("transform-dag-run", dagRunId),
+ ],
+ }),
+ cancelDagRun: builder.mutation<void, TransformDagRunId>({
+ query: (runId) => ({
+ method: "POST",
+ url: `/api/transform-dag-run/${runId}/cancel`,
+ }),
+ invalidatesTags: (_, error, runId) =>
+ invalidateTags(error, [
+ idTag("transform-dag-run", runId),
+ listTag("transform-dag-run"),
+ listTag("transform-run"),
+ tag("transform"),
+ ]),
+ }),
+ cancelJobRun: builder.mutation<void, CancelJobRunRequest>({
+ query: ({ jobId, runId }) => ({
+ method: "POST",
+ url: `/api/transform-job/${jobId}/runs/${runId}/cancel`,
+ }),
+ invalidatesTags: (_, error, { jobId }) =>
+ invalidateTags(error, [
+ idTag("transform-job", jobId),
+ listTag("transform-job"),
+ listTag("transform-run"),
+ tag("transform"),
+ ]),
+ }),
+ listTransformGraphRuns: builder.query<
+ ListTransformGraphRunsResponse,
+ ListTransformGraphRunsRequest
+ >({
+ query: (params) => ({
+ method: "GET",
+ url: "/api/transform/runs",
+ params,
+ }),
+ providesTags: [
+ listTag("transform-run"),
+ listTag("transform-dag-run"),
+ listTag("transform-job"),
+ ],
+ }),
createTransform: builder.mutation<Transform, CreateTransformRequest>({
query: (body) => ({
method: "POST",
@@ -190,7 +286,12 @@ export const transformApi = Api.injectEndpoints({
url: `/api/transform/${id}`,
}),
invalidatesTags: (_, error, id) =>
- invalidateTags(error, [listTag("transform"), idTag("transform", id)]),
+ invalidateTags(error, [
+ listTag("transform"),
+ idTag("transform", id),
+ listTag("transform-run"),
+ listTag("transform-dag-run"),
+ ]),
}),
deleteTransformTarget: builder.mutation<void, TransformId>({
query: (id) => ({
@@ -245,6 +346,12 @@ export const {
useGetInspectorLensQuery,
useLazyGetInspectorLensQuery,
useRunTransformMutation,
+ useRunTransformDagMutation,
+ useListDagTransformsQuery,
+ useListDagRunTransformRunsQuery,
+ useCancelDagRunMutation,
+ useCancelJobRunMutation,
+ useListTransformGraphRunsQuery,
useCancelCurrentTransformRunMutation,
useCreateTransformMutation,
useUpdateTransformMutation,
diff --git a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx
index e6af54d6d90..9d97c0a953d 100644
--- a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx
+++ b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/DataStudioLayout.tsx
@@ -233,7 +233,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
<DataStudioTab
label={t`Runs`}
icon="play_outlined"
- to={Urls.transformRunList()}
+ to={Urls.transformGraphRunList()}
isSelected={currentTab === "runs"}
showLabel={isNavbarOpened}
/>
diff --git a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts
index d0cb7ce5c1d..ee52532f714 100644
--- a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts
+++ b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/utils.ts
@@ -32,7 +32,7 @@ export const getCurrentTab = (pathname: string): TabName => {
return "schema-viewer";
case pathname.startsWith(Urls.dataStudioLibrary()):
return "library";
- case pathname.startsWith(Urls.transformRunList()):
+ case pathname.startsWith(Urls.transformGraphRunList()):
return "runs";
case pathname.startsWith(Urls.transformList()):
return "transforms";
diff --git a/frontend/src/metabase/transforms/analytics.ts b/frontend/src/metabase/transforms/analytics.ts
index d7be9613167..e7aa306069f 100644
--- a/frontend/src/metabase/transforms/analytics.ts
+++ b/frontend/src/metabase/transforms/analytics.ts
@@ -3,6 +3,7 @@ import type {
IndexKind,
InspectorCardId,
InspectorLensId,
+ TransformDagDirection,
TransformId,
TransformJobId,
} from "metabase-types/api";
@@ -21,6 +22,20 @@ export function trackTransformTriggerManualRun({
});
}
+export function trackTransformTriggerDagRun({
+ transformId,
+ direction,
+}: {
+ transformId: TransformId;
+ direction: TransformDagDirection;
+}) {
+ trackSimpleEvent({
+ event: "transform_trigger_dag_run",
+ target_id: transformId,
+ event_detail: direction,
+ });
+}
+
export function trackTransformJobTriggerManualRun({
jobId,
}: {
@@ -32,6 +47,17 @@ export function trackTransformJobTriggerManualRun({
});
}
+export function trackTransformRunsViewToggled({
+ view,
+}: {
+ view: "detailed" | "grouped";
+}) {
+ trackSimpleEvent({
+ event: "transform_runs_view_toggled",
+ event_detail: view,
+ });
+}
+
export function trackTransformCreate({
creationType,
}: {
diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.css b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.css
new file mode 100644
index 00000000000..5b1788f5547
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.module.css
@@ -0,0 +1,3 @@
+.label {
+ white-space: nowrap;
+}
diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsx b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsx
new file mode 100644
index 00000000000..1e5bfee3672
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.tsx
@@ -0,0 +1,44 @@
+import { t } from "ttag";
+
+import { useDispatch } from "metabase/redux";
+import { push } from "metabase/router";
+import { trackTransformRunsViewToggled } from "metabase/transforms/analytics";
+import { Switch } from "metabase/ui";
+import * as Urls from "metabase/urls";
+
+import S from "./DetailedViewSwitch.module.css";
+
+type DetailedViewSwitchProps = {
+ detailed: boolean;
+ params: Urls.CommonRunListParams;
+};
+
+export function DetailedViewSwitch({
+ detailed,
+ params,
+}: DetailedViewSwitchProps) {
+ const dispatch = useDispatch();
+
+ const handleChange = () => {
+ const nextView = detailed ? "grouped" : "detailed";
+ trackTransformRunsViewToggled({ view: nextView });
+ dispatch(
+ push(
+ detailed
+ ? Urls.transformGraphRunList(params)
+ : Urls.transformRunList(params),
+ ),
+ );
+ };
+
+ return (
+ <Switch
+ label={t`Detailed view`}
+ labelPosition="left"
+ checked={detailed}
+ onChange={handleChange}
+ classNames={{ label: S.label }}
+ data-testid="detailed-view-switch"
+ />
+ );
+}
diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsx b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsx
new file mode 100644
index 00000000000..1836d05262b
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/DetailedViewSwitch.unit.spec.tsx
@@ -0,0 +1,63 @@
+import userEvent from "@testing-library/user-event";
+
+import { renderWithProviders, screen } from "__support__/ui";
+import { Route } from "metabase/router";
+import * as Urls from "metabase/urls";
+
+import { trackTransformRunsViewToggled } from "../../analytics";
+
+import { DetailedViewSwitch } from "./DetailedViewSwitch";
+
+jest.mock("../../analytics", () => ({
+ ...jest.requireActual("../../analytics"),
+ trackTransformRunsViewToggled: jest.fn(),
+}));
+
+type SetupOpts = {
+ detailed: boolean;
+ params?: Urls.CommonRunListParams;
+};
+
+function setup({ detailed, params = {} }: SetupOpts) {
+ const initialRoute = detailed
+ ? Urls.transformRunList()
+ : Urls.transformGraphRunList();
+
+ const { history } = renderWithProviders(
+ <Route
+ path="*"
+ element={<DetailedViewSwitch detailed={detailed} params={params} />}
+ />,
+ { withRouter: true, initialRoute },
+ );
+
+ return { history };
+}
+
+describe("DetailedViewSwitch", () => {
+ it("tracks a toggle to the detailed view and navigates there", async () => {
+ const { history } = setup({ detailed: false });
+
+ await userEvent.click(screen.getByLabelText("Detailed view"));
+
+ expect(trackTransformRunsViewToggled).toHaveBeenCalledWith({
+ view: "detailed",
+ });
+ expect(history?.getCurrentLocation().pathname).toBe(
+ "/data-studio/transforms/runs/individual",
+ );
+ });
+
+ it("tracks a toggle to the grouped view and navigates there", async () => {
+ const { history } = setup({ detailed: true });
+
+ await userEvent.click(screen.getByLabelText("Detailed view"));
+
+ expect(trackTransformRunsViewToggled).toHaveBeenCalledWith({
+ view: "grouped",
+ });
+ expect(history?.getCurrentLocation().pathname).toBe(
+ "/data-studio/transforms/runs",
+ );
+ });
+});
diff --git a/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.ts b/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.ts
new file mode 100644
index 00000000000..ea1b634ed75
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/DetailedViewSwitch/index.ts
@@ -0,0 +1 @@
+export { DetailedViewSwitch } from "./DetailedViewSwitch";
diff --git a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx
index c3c2262e22c..8234d3b8e19 100644
--- a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx
+++ b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.tsx
@@ -1,12 +1,18 @@
+import { useDisclosure } from "@mantine/hooks";
import { c, t } from "ttag";
-import { useRunTransformJobMutation } from "metabase/api";
+import {
+ useCancelJobRunMutation,
+ useRunTransformJobMutation,
+} from "metabase/api";
+import { ConfirmModal } from "metabase/common/components/ConfirmModal";
import { Schedule } from "metabase/common/components/Schedule";
import { TitleSection } from "metabase/common/data-studio/components/TitleSection";
import { useSetting } from "metabase/common/hooks";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { Box, Divider, Group, Tooltip } from "metabase/ui";
import { getScheduleExplanation } from "metabase/utils/cron";
+import { isResourceNotFoundError } from "metabase/utils/errors";
import type {
ScheduleDisplayType,
ScheduleSettings,
@@ -130,7 +136,12 @@ function RunButtonSection({
isCheckingPermissions,
}: RunButtonSectionProps) {
const [runJob] = useRunTransformJobMutation();
+ const [cancelJobRun] = useCancelJobRunMutation();
const { sendErrorToast } = useMetadataToasts();
+ const [
+ isCancelModalOpen,
+ { open: openCancelModal, close: closeCancelModal },
+ ] = useDisclosure(false);
const isSaved = job.id != null;
const hasTags = job.tag_ids?.length !== 0;
@@ -160,15 +171,44 @@ function RunButtonSection({
}
};
+ const handleCancel = async () => {
+ if (job.id == null || job.last_run?.id == null) {
+ return;
+ }
+ const { error } = await cancelJobRun({
+ jobId: job.id,
+ runId: job.last_run.id,
+ });
+ if (error && !isResourceNotFoundError(error)) {
+ sendErrorToast(t`Failed to cancel job`);
+ }
+ };
+
return (
- <Tooltip label={tooltipLabel} disabled={!tooltipLabel}>
- <RunButton
- id={job.id}
- run={job.last_run}
- isDisabled={!isSaved || !hasTags || readOnly}
- isLoading={isCheckingPermissions}
- onRun={handleRun}
+ <>
+ <Tooltip label={tooltipLabel} disabled={!tooltipLabel}>
+ <RunButton
+ id={job.id}
+ run={job.last_run}
+ isDisabled={!isSaved || !hasTags || readOnly}
+ isLoading={isCheckingPermissions}
+ allowCancellation
+ onRun={handleRun}
+ onCancel={openCancelModal}
+ />
+ </Tooltip>
+ <ConfirmModal
+ title={t`Cancel this run?`}
+ message={t`This stops the job run and requests cancellation of any transforms still in progress. Transforms that have already finished won't be reverted.`}
+ confirmButtonText={t`Cancel run`}
+ closeButtonText={t`Keep running`}
+ opened={isCancelModalOpen}
+ onClose={closeCancelModal}
+ onConfirm={() => {
+ void handleCancel();
+ closeCancelModal();
+ }}
/>
- </Tooltip>
+ </>
);
}
diff --git a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx
index 830b40e0cfc..3b5917825e3 100644
--- a/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/components/JobEditor/ScheduleSection/ScheduleSection.unit.spec.tsx
@@ -1,10 +1,12 @@
import userEvent from "@testing-library/user-event";
+import fetchMock from "fetch-mock";
import {
+ setupCancelJobRunEndpoint,
setupGetTransformJobEndpoint,
setupRunTransformJobEndpoint,
} from "__support__/server-mocks";
-import { renderWithProviders, screen } from "__support__/ui";
+import { renderWithProviders, screen, waitFor } from "__support__/ui";
import type { TransformJob } from "metabase-types/api";
import {
createMockTransformJob,
@@ -81,6 +83,28 @@ describe("ScheduleSection", () => {
).toBeInTheDocument();
});
+ it("should allow cancelling a running job", async () => {
+ const job = createMockTransformJob({
+ id: 3,
+ last_run: createMockTransformRun({ id: 42, status: "started" }),
+ });
+ setupCancelJobRunEndpoint(3, 42);
+ setup({ job });
+
+ await userEvent.click(screen.getByTestId("cancel-button"));
+ await userEvent.click(
+ await screen.findByRole("button", { name: "Cancel run" }),
+ );
+
+ await waitFor(() => {
+ expect(
+ fetchMock.callHistory.called(
+ "path:/api/transform-job/3/runs/42/cancel",
+ ),
+ ).toBe(true);
+ });
+ });
+
it("should handle job run failures", async () => {
const { rerender, onScheduleChange } = setup();
const timestamp = new Date().toISOString();
diff --git a/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx b/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx
index 19ecad2798c..819b352dd73 100644
--- a/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx
+++ b/frontend/src/metabase/transforms/components/RunButton/RunButton.tsx
@@ -8,7 +8,14 @@ import {
import { t } from "ttag";
import { useSetting } from "metabase/common/hooks";
-import { Button, type ButtonProps, Icon, Loader, Tooltip } from "metabase/ui";
+import {
+ Button,
+ type ButtonProps,
+ Icon,
+ Loader,
+ Menu,
+ Tooltip,
+} from "metabase/ui";
import type { ColorName } from "metabase/ui/colors/types";
import type {
TransformId,
@@ -16,6 +23,7 @@ import type {
TransformRun,
} from "metabase-types/api";
+import { isActiveRunStatus } from "../../utils";
import { LockedTransformsHoverCard } from "../LockedTransformsHoverCard/LockedTransformsHoverCard";
const RECENT_TIMEOUT = 5000;
@@ -29,6 +37,7 @@ type RunButtonProps = {
size?: ButtonProps["size"];
onRun: () => void;
onCancel?: () => void;
+ menuItems?: ReactNode;
};
export const RunButton = forwardRef(function RunButton(
@@ -41,6 +50,7 @@ export const RunButton = forwardRef(function RunButton(
size = "md",
onRun,
onCancel,
+ menuItems,
}: RunButtonProps,
ref: Ref<HTMLButtonElement>,
) {
@@ -63,6 +73,9 @@ export const RunButton = forwardRef(function RunButton(
setIsRecent(false);
}, [id]);
+ const isRunning = isActiveRunStatus(run?.status);
+ const showMenu = menuItems != null && !isRunning;
+
const runButton = (
<Button.Group>
<Button
@@ -77,6 +90,24 @@ export const RunButton = forwardRef(function RunButton(
>
{label}
</Button>
+ {showMenu && (
+ <Menu position="bottom-end">
+ <Menu.Target>
+ <Button
+ variant="filled"
+ color={color}
+ disabled={isDisabled}
+ size={size}
+ px="sm"
+ aria-label={t`More run options`}
+ data-testid="run-options-button"
+ >
+ <Icon name="chevrondown" aria-hidden />
+ </Button>
+ </Menu.Target>
+ <Menu.Dropdown>{menuItems}</Menu.Dropdown>
+ </Menu>
+ )}
{allowCancellation && run?.status === "started" && (
<Tooltip label={t`Cancel`}>
<Button
diff --git a/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx b/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx
index d92cc62fdaf..2549973b128 100644
--- a/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/components/RunButton/RunButton.unit.spec.tsx
@@ -1,21 +1,36 @@
+import type { ReactNode } from "react";
+
import { mockSettings } from "__support__/settings";
import { act, fireEvent, renderWithProviders, screen } from "__support__/ui";
import { createMockState } from "metabase/redux/store/mocks";
+import { Menu } from "metabase/ui";
+import type { TransformRun } from "metabase-types/api";
+import { createMockTransformRun } from "metabase-types/api/mocks";
import { RunButton } from "./RunButton";
function setup({
isMeterLocked = null,
+ run = null,
+ menuItems,
+ onRun = jest.fn(),
}: {
isMeterLocked?: boolean | null;
+ run?: TransformRun | null;
+ menuItems?: ReactNode;
+ onRun?: () => void;
} = {}) {
jest.useFakeTimers();
const state = createMockState({
settings: mockSettings({ "transforms-meter-locked": isMeterLocked }),
});
- renderWithProviders(<RunButton id={1} run={null} onRun={jest.fn()} />, {
- storeInitialState: state,
- });
+ renderWithProviders(
+ <RunButton id={1} run={run} onRun={onRun} menuItems={menuItems} />,
+ {
+ storeInitialState: state,
+ },
+ );
+ return { onRun };
}
describe("RunButton", () => {
@@ -38,6 +53,32 @@ describe("RunButton", () => {
});
});
+ it("calls onRun when the run button is clicked", () => {
+ const { onRun } = setup();
+
+ fireEvent.click(screen.getByTestId("run-button"));
+
+ expect(onRun).toHaveBeenCalledTimes(1);
+ });
+
+ describe("with menu items", () => {
+ const menuItems = <Menu.Item>{"Run upstream"}</Menu.Item>;
+
+ it("shows the run options menu toggle when idle", () => {
+ setup({ menuItems });
+
+ expect(screen.getByTestId("run-options-button")).toBeInTheDocument();
+ });
+
+ it("hides the run options menu toggle while a run is in progress", () => {
+ setup({ menuItems, run: createMockTransformRun({ status: "started" }) });
+
+ expect(
+ screen.queryByTestId("run-options-button"),
+ ).not.toBeInTheDocument();
+ });
+ });
+
describe("when transforms are locked", () => {
it("renders a disabled run button and shows an explanatory card on hover", async () => {
setup({ isMeterLocked: true });
diff --git a/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx b/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx
index c9c5e19cdcf..100e8ca7ac2 100644
--- a/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx
+++ b/frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx
@@ -13,12 +13,13 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErr
import { DataStudioBreadcrumbs } from "metabase/common/data-studio/components/DataStudioBreadcrumbs";
import { PaneHeader } from "metabase/common/data-studio/components/PaneHeader";
import { useSetting } from "metabase/common/hooks";
-import { usePageTitle } from "metabase/hooks/use-page-title";
import { useDispatch } from "metabase/redux";
import { replace } from "metabase/router";
+import { DetailedViewSwitch } from "metabase/transforms/components/DetailedViewSwitch";
import { LockedTransformsBanner } from "metabase/transforms/components/LockedTransformsBanner/LockedTransformsBanner";
import { POLLING_INTERVAL } from "metabase/transforms/constants";
-import { Center, Flex, Stack } from "metabase/ui";
+import { isActiveRunStatus } from "metabase/transforms/utils";
+import { Center, Flex, Group, Stack } from "metabase/ui";
import * as Urls from "metabase/urls";
import type { TransformRun, TransformRunId } from "metabase-types/api";
@@ -46,7 +47,6 @@ type RunListPageProps = {
};
export function RunListPage({ location }: RunListPageProps) {
- usePageTitle(t`Runs`);
const params = getParsedParams(location);
const { page = 0 } = params;
const { ref: containerRef, width: containerWidth } = useElementSize();
@@ -173,14 +173,20 @@ export function RunListPage({ location }: RunListPageProps) {
<LoadingAndErrorWrapper loading={isLoading} error={error} />
</Center>
) : (
- <Stack flex="0 1 auto" mih={0} gap="lg">
+ <Stack flex="0 1 auto" mih={0} gap="lg" pt="2.5rem">
{isMeterLocked && <LockedTransformsBanner />}
- <RunFilterBar
- filterOptions={getFilterOptions(params)}
- transforms={transforms}
- tags={tags}
- onFilterOptionsChange={handleFilterOptionsChange}
- />
+ <Group justify="space-between" align="center" wrap="nowrap">
+ <RunFilterBar
+ filterOptions={getFilterOptions(params)}
+ transforms={transforms}
+ tags={tags}
+ onFilterOptionsChange={handleFilterOptionsChange}
+ />
+ <DetailedViewSwitch
+ detailed={true}
+ params={Urls.pickCommonRunListParams(params)}
+ />
+ </Group>
<RunTable
runs={runs}
tags={tags}
@@ -212,7 +218,5 @@ export function RunListPage({ location }: RunListPageProps) {
}
export function isPollingNeeded(runs: TransformRun[] = []) {
- return runs.some(
- (run) => run.status === "started" || run.status === "canceling",
- );
+ return runs.some((run) => isActiveRunStatus(run.status));
}
diff --git a/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsx b/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsx
new file mode 100644
index 00000000000..0caa006fa38
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/RunsPage/RunsPage.tsx
@@ -0,0 +1,15 @@
+import { t } from "ttag";
+
+import { usePageTitle } from "metabase/hooks/use-page-title";
+import { Outlet } from "metabase/router";
+import { Flex } from "metabase/ui";
+
+export function RunsPage() {
+ usePageTitle(t`Runs`);
+
+ return (
+ <Flex direction="column" w="100%" h="100%">
+ <Outlet />
+ </Flex>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/RunsPage/index.ts b/frontend/src/metabase/transforms/pages/RunsPage/index.ts
new file mode 100644
index 00000000000..e27f52d10ad
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/RunsPage/index.ts
@@ -0,0 +1 @@
+export { RunsPage } from "./RunsPage";
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsx
new file mode 100644
index 00000000000..19ca92f6376
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TransformGraphRunFilterBar.tsx
@@ -0,0 +1,97 @@
+import { t } from "ttag";
+
+import type { RelativeIntervalDirection } from "metabase/querying/common/types";
+import { RunMethodFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/RunMethodFilterWidget";
+import { StatusFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/StatusFilterWidget";
+import { TimeFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/TimeFilterWidget";
+import { TransformFilterWidget } from "metabase/transforms/pages/RunListPage/RunFilterBar/TransformFilterWidget";
+import { Group } from "metabase/ui";
+import type {
+ Transform,
+ TransformGraphRunType,
+ TransformId,
+ TransformRunMethod,
+ TransformRunStatus,
+} from "metabase-types/api";
+
+import type { TransformGraphRunFilterOptions } from "../types";
+
+import { TypeFilterWidget } from "./TypeFilterWidget";
+
+const PAST_INTERVAL_DIRECTIONS: RelativeIntervalDirection[] = [
+ "past",
+ "current",
+];
+
+type TransformGraphRunFilterBarProps = {
+ filterOptions: TransformGraphRunFilterOptions;
+ transforms: Transform[];
+ onFilterOptionsChange: (
+ filterOptions: TransformGraphRunFilterOptions,
+ ) => void;
+};
+
+export function TransformGraphRunFilterBar({
+ filterOptions,
+ transforms,
+ onFilterOptionsChange,
+}: TransformGraphRunFilterBarProps) {
+ const handleTypesChange = (types: TransformGraphRunType[]) => {
+ onFilterOptionsChange({ ...filterOptions, types });
+ };
+
+ const handleTransformsChange = (transformIds: TransformId[]) => {
+ onFilterOptionsChange({ ...filterOptions, transformIds });
+ };
+
+ const handleStatusesChange = (statuses: TransformRunStatus[]) => {
+ onFilterOptionsChange({ ...filterOptions, statuses });
+ };
+
+ const handleStartTimeChange = (startTime: string | undefined) => {
+ onFilterOptionsChange({ ...filterOptions, startTime });
+ };
+
+ const handleEndTimeChange = (endTime: string | undefined) => {
+ onFilterOptionsChange({ ...filterOptions, endTime });
+ };
+
+ const handleRunMethodsChange = (runMethods: TransformRunMethod[]) => {
+ onFilterOptionsChange({ ...filterOptions, runMethods });
+ };
+
+ return (
+ <Group>
+ <TypeFilterWidget
+ types={filterOptions.types ?? []}
+ onChange={handleTypesChange}
+ />
+ <TransformFilterWidget
+ transformIds={filterOptions.transformIds ?? []}
+ transforms={transforms}
+ onChange={handleTransformsChange}
+ />
+ <TimeFilterWidget
+ label={t`Started at`}
+ value={filterOptions.startTime}
+ availableDirections={PAST_INTERVAL_DIRECTIONS}
+ onChange={handleStartTimeChange}
+ />
+ <TimeFilterWidget
+ label={t`Ended at`}
+ value={filterOptions.endTime}
+ availableDirections={PAST_INTERVAL_DIRECTIONS}
+ onChange={handleEndTimeChange}
+ />
+ <StatusFilterWidget
+ label={t`Status`}
+ statuses={filterOptions.statuses ?? []}
+ onChange={handleStatusesChange}
+ />
+ <RunMethodFilterWidget
+ runMethods={filterOptions.runMethods ?? []}
+ onChange={handleRunMethodsChange}
+ />
+ </Group>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsx
new file mode 100644
index 00000000000..4d4d43fcb92
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/TypeFilterWidget.tsx
@@ -0,0 +1,124 @@
+import { useDisclosure } from "@mantine/hooks";
+import { type FormEvent, useState } from "react";
+import { t } from "ttag";
+
+import { FILTER_WIDGET_MIN_WIDTH } from "metabase/transforms/constants";
+import { FilterFieldSet } from "metabase/transforms/pages/RunListPage/RunFilterBar/FilterFieldSet";
+import {
+ Box,
+ Button,
+ Checkbox,
+ Divider,
+ Group,
+ Popover,
+ Stack,
+} from "metabase/ui";
+import {
+ TRANSFORM_GRAPH_RUN_TYPES,
+ type TransformGraphRunType,
+} from "metabase-types/api";
+
+export function formatGraphRunType(type: TransformGraphRunType): string {
+ switch (type) {
+ case "job":
+ return t`Job`;
+ case "dag":
+ return t`Upstream / downstream`;
+ case "transform":
+ return t`Single transformation`;
+ }
+}
+
+function isTransformGraphRunType(
+ value: string,
+): value is TransformGraphRunType {
+ return TRANSFORM_GRAPH_RUN_TYPES.some((type) => type === value);
+}
+
+type TypeFilterWidgetProps = {
+ types: TransformGraphRunType[];
+ onChange: (types: TransformGraphRunType[]) => void;
+};
+
+export function TypeFilterWidget({ types, onChange }: TypeFilterWidgetProps) {
+ const [isOpened, { toggle, close }] = useDisclosure();
+
+ const handleRemove = () => {
+ onChange([]);
+ };
+
+ const handleSubmit = (nextTypes: TransformGraphRunType[]) => {
+ close();
+ onChange(nextTypes);
+ };
+
+ return (
+ <Popover opened={isOpened} position="bottom-start" onDismiss={close}>
+ <Popover.Target>
+ <FilterFieldSet
+ label={t`Type`}
+ icon="list"
+ displayValue={getDisplayValue(types)}
+ onClick={toggle}
+ onRemove={handleRemove}
+ />
+ </Popover.Target>
+ <Popover.Dropdown>
+ <TypeFilterForm types={types} onSubmit={handleSubmit} />
+ </Popover.Dropdown>
+ </Popover>
+ );
+}
+
+function getDisplayValue(types: TransformGraphRunType[]) {
+ if (types.length === 0) {
+ return null;
+ }
+ return types.map(formatGraphRunType).join(", ");
+}
+
+type TypeFilterFormProps = {
+ types: TransformGraphRunType[];
+ onSubmit: (types: TransformGraphRunType[]) => void;
+};
+
+function TypeFilterForm({
+ types: initialTypes,
+ onSubmit,
+}: TypeFilterFormProps) {
+ const [types, setTypes] = useState(initialTypes);
+ const isValid = types.length > 0;
+
+ const handleChange = (values: string[]) => {
+ setTypes(values.filter(isTransformGraphRunType));
+ };
+
+ const handleSubmit = (event: FormEvent) => {
+ event.preventDefault();
+ if (isValid) {
+ onSubmit(types);
+ }
+ };
+
+ return (
+ <Box component="form" miw={FILTER_WIDGET_MIN_WIDTH} onSubmit={handleSubmit}>
+ <Checkbox.Group value={types} onChange={handleChange}>
+ <Stack p="md">
+ {TRANSFORM_GRAPH_RUN_TYPES.map((type) => (
+ <Checkbox
+ key={type}
+ value={type}
+ label={formatGraphRunType(type)}
+ />
+ ))}
+ </Stack>
+ </Checkbox.Group>
+ <Divider />
+ <Group p="md" justify="end">
+ <Button type="submit" variant="filled" disabled={!isValid}>
+ {initialTypes.length > 0 ? t`Update filter` : t`Add filter`}
+ </Button>
+ </Group>
+ </Box>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.ts
new file mode 100644
index 00000000000..1062980da59
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunFilterBar/index.ts
@@ -0,0 +1 @@
+export { TransformGraphRunFilterBar } from "./TransformGraphRunFilterBar";
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.css b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.css
new file mode 100644
index 00000000000..5780d690fe1
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.module.css
@@ -0,0 +1,7 @@
+.main {
+ overflow: hidden;
+}
+
+.resizing {
+ user-select: none;
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsx
new file mode 100644
index 00000000000..3f18761e0b7
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.tsx
@@ -0,0 +1,209 @@
+import { useDisclosure, useElementSize } from "@mantine/hooks";
+import cx from "classnames";
+import type { Location } from "history";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { t } from "ttag";
+
+import {
+ useListTransformGraphRunsQuery,
+ useListTransformsQuery,
+} from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { PaginationControls } from "metabase/common/components/PaginationControls";
+import { DataStudioBreadcrumbs } from "metabase/common/data-studio/components/DataStudioBreadcrumbs";
+import { PaneHeader } from "metabase/common/data-studio/components/PaneHeader";
+import { useDispatch } from "metabase/redux";
+import { replace } from "metabase/router";
+import { DetailedViewSwitch } from "metabase/transforms/components/DetailedViewSwitch";
+import { POLLING_INTERVAL } from "metabase/transforms/constants";
+import { isActiveRunStatus } from "metabase/transforms/utils";
+import { Center, Flex, Group, Stack } from "metabase/ui";
+import * as Urls from "metabase/urls";
+import type { TransformGraphRun } from "metabase-types/api";
+
+import { TransformGraphRunFilterBar } from "./TransformGraphRunFilterBar";
+import S from "./TransformGraphRunListPage.module.css";
+import { TransformGraphRunSidebar } from "./TransformGraphRunSidebar";
+import { TransformGraphRunTable, getRowKey } from "./TransformGraphRunTable";
+import { PAGE_SIZE } from "./constants";
+import type {
+ TransformGraphRunFilterOptions,
+ TransformGraphRunSortOptions,
+} from "./types";
+import {
+ getFilterOptions,
+ getParsedParams,
+ getSortOptions,
+ hasFilterOptions,
+} from "./utils";
+
+const EMPTY_RUNS: TransformGraphRun[] = [];
+
+type TransformGraphRunListPageProps = {
+ location: Location;
+};
+
+export function TransformGraphRunListPage({
+ location,
+}: TransformGraphRunListPageProps) {
+ const params = useMemo(() => getParsedParams(location), [location]);
+ const filterOptions = useMemo(() => getFilterOptions(params), [params]);
+ const { page = 0 } = params;
+ const { ref: containerRef, width: containerWidth } = useElementSize();
+ const [isResizing, { open: startResizing, close: stopResizing }] =
+ useDisclosure();
+ const [selectedRun, setSelectedRun] = useState<TransformGraphRun>();
+ const [isPolling, setIsPolling] = useState(false);
+ const dispatch = useDispatch();
+
+ const {
+ data,
+ isLoading: isLoadingRuns,
+ error,
+ } = useListTransformGraphRunsQuery(
+ {
+ offset: page * PAGE_SIZE,
+ limit: PAGE_SIZE,
+ types: params.types,
+ statuses: params.statuses,
+ "transform-ids": params.transformIds,
+ "start-time": params.startTime,
+ "end-time": params.endTime,
+ "run-methods": params.runMethods,
+ "sort-column": params.sortColumn,
+ "sort-direction": params.sortDirection,
+ },
+ {
+ pollingInterval: isPolling ? POLLING_INTERVAL : undefined,
+ },
+ );
+
+ const { data: transforms = [], isLoading: isLoadingTransforms } =
+ useListTransformsQuery({});
+
+ const isLoading = isLoadingRuns || isLoadingTransforms;
+
+ if (isPolling !== isPollingNeeded(data?.data)) {
+ setIsPolling(isPollingNeeded(data?.data));
+ }
+
+ const runs = data?.data ?? EMPTY_RUNS;
+
+ useEffect(() => {
+ setSelectedRun((current) =>
+ current == null
+ ? current
+ : // Do not close sidebar on its own if selected run goes to next page
+ (runs.find((run) => getRowKey(run) === getRowKey(current)) ??
+ current),
+ );
+ }, [runs]);
+
+ const handleParamsChange = useCallback(
+ (newParams: Urls.TransformGraphRunListParams) => {
+ dispatch(replace(Urls.transformGraphRunList(newParams)));
+ },
+ [dispatch],
+ );
+
+ const handleFilterOptionsChange = useCallback(
+ (filterOptions: TransformGraphRunFilterOptions) => {
+ handleParamsChange({ ...params, ...filterOptions, page: undefined });
+ },
+ [params, handleParamsChange],
+ );
+
+ const handleSortOptionsChange = useCallback(
+ (sortOptions: TransformGraphRunSortOptions | undefined) => {
+ handleParamsChange({
+ ...params,
+ sortColumn: sortOptions?.column,
+ sortDirection: sortOptions?.direction,
+ page: undefined,
+ });
+ },
+ [params, handleParamsChange],
+ );
+
+ const handlePageChange = useCallback(
+ (nextPage: number) => {
+ handleParamsChange({ ...params, page: nextPage });
+ },
+ [params, handleParamsChange],
+ );
+
+ const handleSelect = useCallback(
+ (run: TransformGraphRun) => setSelectedRun(run),
+ [],
+ );
+
+ return (
+ <Flex
+ className={cx({ [S.resizing]: isResizing })}
+ ref={containerRef}
+ h="100%"
+ wrap="nowrap"
+ data-testid="transform-graph-run-list"
+ >
+ <Stack className={S.main} flex={1} px="3.5rem" pb="md" gap={0}>
+ <PaneHeader
+ breadcrumbs={<DataStudioBreadcrumbs>{t`Runs`}</DataStudioBreadcrumbs>}
+ py={0}
+ showMetabotButton
+ />
+ {isLoading || error != null ? (
+ <Center h="100%">
+ <LoadingAndErrorWrapper loading={isLoading} error={error} />
+ </Center>
+ ) : (
+ <Stack flex="0 1 auto" mih={0} gap="lg" pt="2.5rem">
+ <Group justify="space-between" align="center" wrap="nowrap">
+ <TransformGraphRunFilterBar
+ filterOptions={filterOptions}
+ transforms={transforms}
+ onFilterOptionsChange={handleFilterOptionsChange}
+ />
+ <DetailedViewSwitch
+ detailed={false}
+ params={Urls.pickCommonRunListParams(params)}
+ />
+ </Group>
+ <TransformGraphRunTable
+ runs={runs}
+ hasFilters={hasFilterOptions(filterOptions)}
+ sortOptions={getSortOptions(params)}
+ onSortOptionsChange={handleSortOptionsChange}
+ onSelect={handleSelect}
+ />
+ {data != null && data.total > PAGE_SIZE && (
+ <Group justify="end">
+ <PaginationControls
+ page={page}
+ pageSize={PAGE_SIZE}
+ itemsLength={runs.length}
+ total={data.total}
+ showTotal
+ onPreviousPage={() => handlePageChange(page - 1)}
+ onNextPage={() => handlePageChange(page + 1)}
+ />
+ </Group>
+ )}
+ </Stack>
+ )}
+ </Stack>
+ {selectedRun != null && (
+ <TransformGraphRunSidebar
+ run={selectedRun}
+ containerWidth={containerWidth}
+ onResizeStart={startResizing}
+ onResizeStop={stopResizing}
+ onClose={() => setSelectedRun(undefined)}
+ />
+ )}
+ </Flex>
+ );
+}
+
+export function isPollingNeeded(runs: TransformGraphRun[] = []) {
+ return runs.some((run) => isActiveRunStatus(run.status));
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsx
new file mode 100644
index 00000000000..b56c89d6ad3
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunListPage.unit.spec.tsx
@@ -0,0 +1,264 @@
+import userEvent from "@testing-library/user-event";
+import fetchMock from "fetch-mock";
+
+import {
+ setupCancelJobRunEndpoint,
+ setupListDagRunTransformRunsEndpoint,
+ setupListJobRunTransformRunsEndpoint,
+ setupListTransformGraphRunsEndpoint,
+ setupListTransformsEndpoint,
+ setupUserMetabotPermissionsEndpoint,
+} from "__support__/server-mocks";
+import {
+ mockGetBoundingClientRect,
+ renderWithProviders,
+ screen,
+ waitFor,
+ within,
+} from "__support__/ui";
+import { Route, withRouteProps } from "metabase/router";
+import type { TransformGraphRun } from "metabase-types/api";
+import {
+ createMockListTransformGraphRunsResponse,
+ createMockTransformGraphRun,
+ createMockTransformRunForJobRun,
+} from "metabase-types/api/mocks";
+
+import { TransformGraphRunListPage } from "./TransformGraphRunListPage";
+
+const RoutedTransformGraphRunListPage = withRouteProps(
+ TransformGraphRunListPage,
+);
+
+type SetupOpts = {
+ runs?: TransformGraphRun[];
+ initialRoute?: string;
+};
+
+function setup({
+ runs = [],
+ initialRoute = "/data-studio/transforms/runs",
+}: SetupOpts = {}) {
+ setupUserMetabotPermissionsEndpoint();
+ setupListTransformsEndpoint([]);
+ setupListTransformGraphRunsEndpoint(
+ createMockListTransformGraphRunsResponse({
+ data: runs,
+ total: runs.length,
+ }),
+ );
+ mockGetBoundingClientRect({ width: 1200, height: 800 });
+
+ const { history } = renderWithProviders(
+ <Route
+ path="/data-studio/transforms/runs"
+ element={<RoutedTransformGraphRunListPage />}
+ />,
+ { withRouter: true, initialRoute },
+ );
+
+ return { history };
+}
+
+const JOB_RUN = createMockTransformGraphRun({
+ run_type: "job",
+ id: 101,
+ entity_id: 1,
+ name: "Hourly refresh",
+ run_method: "cron",
+});
+
+const DAG_RUN = createMockTransformGraphRun({
+ run_type: "dag",
+ id: 202,
+ entity_id: 12,
+ name: "Customers deduped",
+ direction: "upstream",
+ transform_count: 5,
+ status: "failed",
+});
+
+const TRANSFORM_RUN = createMockTransformGraphRun({
+ run_type: "transform",
+ id: 301,
+ entity_id: 20,
+ name: "Products normalized",
+});
+
+describe("TransformGraphRunListPage", () => {
+ it("renders job, DAG, and standalone runs with a Run name and Type", async () => {
+ setup({ runs: [JOB_RUN, DAG_RUN, TRANSFORM_RUN] });
+
+ expect(await screen.findByText("Hourly refresh")).toBeInTheDocument();
+ expect(screen.getByText("4 dependencies")).toBeInTheDocument();
+ expect(screen.getByText("Customers deduped")).toBeInTheDocument();
+ expect(screen.getByText("Products normalized")).toBeInTheDocument();
+
+ const table = screen.getByRole("treegrid", { name: "Runs" });
+ expect(within(table).getByText("Job")).toBeInTheDocument();
+ expect(within(table).getAllByText("Transformation")).toHaveLength(2);
+ });
+
+ it("shows the Detailed view switch unchecked in the grouped view", async () => {
+ setup({ runs: [] });
+ expect(await screen.findByLabelText("Detailed view")).not.toBeChecked();
+ });
+
+ it("preserves shared filters (and drops view-specific params) when toggling to the detailed view", async () => {
+ const { history } = setup({
+ runs: [],
+ initialRoute:
+ "/data-studio/transforms/runs?statuses=failed&transform-ids=7&start-time=2024-01-01&run-methods=cron&sort-column=start_time&sort-direction=desc&page=2",
+ });
+
+ await userEvent.click(await screen.findByLabelText("Detailed view"));
+
+ const location = history?.getCurrentLocation();
+ expect(location?.pathname).toBe("/data-studio/transforms/runs/individual");
+
+ const search = new URLSearchParams(location?.search);
+ expect(search.getAll("statuses")).toEqual(["failed"]);
+ expect(search.getAll("transform-ids")).toEqual(["7"]);
+ expect(search.get("start-time")).toBe("2024-01-01");
+ expect(search.getAll("run-methods")).toEqual(["cron"]);
+
+ expect(search.has("sort-column")).toBe(false);
+ expect(search.has("page")).toBe(false);
+ });
+
+ it("greys a deleted-entity run and hides the Transforms section for a deleted job", async () => {
+ const deletedJobRun = createMockTransformGraphRun({
+ run_type: "job",
+ id: 103,
+ entity_id: null,
+ name: "Removed job",
+ run_method: "cron",
+ });
+ setup({ runs: [deletedJobRun] });
+
+ const name = await screen.findByText("Removed job");
+ await userEvent.hover(name);
+ expect(
+ await screen.findByText("Removed job has been deleted"),
+ ).toBeInTheDocument();
+
+ await userEvent.click(name);
+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");
+
+ expect(within(sidebar).queryByText("Transforms")).not.toBeInTheDocument();
+ expect(
+ within(sidebar).queryByRole("link", { name: "View this job" }),
+ ).not.toBeInTheDocument();
+ });
+
+ it("still shows member transforms for a DAG run whose seed transform was deleted", async () => {
+ const deletedDagRun = createMockTransformGraphRun({
+ run_type: "dag",
+ id: 205,
+ entity_id: null,
+ name: "Removed seed",
+ direction: "upstream",
+ transform_count: 3,
+ status: "succeeded",
+ });
+ setupListDagRunTransformRunsEndpoint(205, [
+ createMockTransformRunForJobRun({
+ id: 1,
+ transform_name: "Orders cleaned",
+ status: "succeeded",
+ }),
+ ]);
+ setup({ runs: [deletedDagRun] });
+
+ await userEvent.click(await screen.findByText("Removed seed"));
+
+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");
+
+ expect(within(sidebar).getByText("Transforms")).toBeInTheDocument();
+ expect(
+ await within(sidebar).findByText("Orders cleaned"),
+ ).toBeInTheDocument();
+ });
+
+ it("drills into a DAG run's member transforms via the dag-run endpoint", async () => {
+ setupListDagRunTransformRunsEndpoint(202, [
+ createMockTransformRunForJobRun({
+ id: 1,
+ transform_name: "Orders cleaned",
+ status: "succeeded",
+ }),
+ createMockTransformRunForJobRun({
+ id: 2,
+ transform_name: "Customers deduped",
+ status: "failed",
+ message: "boom",
+ }),
+ ]);
+ setup({ runs: [DAG_RUN] });
+
+ await userEvent.click(await screen.findByText("Customers deduped"));
+
+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");
+ expect(
+ await within(sidebar).findByText("Orders cleaned"),
+ ).toBeInTheDocument();
+
+ expect(
+ within(sidebar).getByRole("region", { name: "Error" }),
+ ).toBeInTheDocument();
+
+ expect(
+ within(sidebar).getByRole("link", { name: "View this transform" }),
+ ).toHaveAttribute("href", "/data-studio/transforms/12");
+ });
+
+ it("shows a standalone transform run as its own single member (no fetch)", async () => {
+ setup({ runs: [TRANSFORM_RUN] });
+
+ await userEvent.click(await screen.findByText("Products normalized"));
+
+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");
+ const member = within(sidebar).getByTestId("transform-run-item");
+ expect(member).toHaveTextContent("Products normalized");
+ });
+
+ it("cancels an in-progress job run via the job-run cancel endpoint", async () => {
+ const startedJob = createMockTransformGraphRun({
+ run_type: "job",
+ id: 102,
+ entity_id: 2,
+ name: "Nightly rollups",
+ run_method: "cron",
+ status: "started",
+ });
+ setupListJobRunTransformRunsEndpoint(2, 102, [
+ createMockTransformRunForJobRun({ id: 5, transform_name: "Step" }),
+ ]);
+ setupCancelJobRunEndpoint(2, 102);
+ setup({ runs: [startedJob] });
+
+ await userEvent.click(await screen.findByText("Nightly rollups"));
+
+ const sidebar = await screen.findByTestId("transform-graph-run-sidebar");
+ expect(
+ within(sidebar).getByRole("link", { name: "View this job" }),
+ ).toHaveAttribute("href", "/data-studio/transforms/jobs/2");
+
+ await userEvent.click(
+ within(sidebar).getByRole("button", { name: "Cancel run" }),
+ );
+
+ const dialog = await screen.findByRole("dialog");
+ await userEvent.click(
+ within(dialog).getByRole("button", { name: "Cancel run" }),
+ );
+
+ await waitFor(() => {
+ expect(
+ fetchMock.callHistory.called(
+ "path:/api/transform-job/2/runs/102/cancel",
+ ),
+ ).toBe(true);
+ });
+ });
+});
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsx
new file mode 100644
index 00000000000..0a76dbd9726
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunInfoSection.tsx
@@ -0,0 +1,55 @@
+import { t } from "ttag";
+
+import { DateTime } from "metabase/common/components/DateTime";
+import { SidebarInfoRow } from "metabase/transforms/components/SidebarInfoRow";
+import {
+ formatRunMethod,
+ formatStatus,
+ getRunDurationMs,
+ isErrorStatus,
+} from "metabase/transforms/utils";
+import { Box, Card } from "metabase/ui";
+import { EMPTY_CELL_PLACEHOLDER } from "metabase/utils/constants";
+import { formatDurationLong } from "metabase/utils/formatting/time";
+import type { TransformGraphRun } from "metabase-types/api";
+
+import { formatRunType } from "../TransformGraphRunTable";
+
+type TransformGraphRunInfoSectionProps = {
+ run: TransformGraphRun;
+};
+
+export function TransformGraphRunInfoSection({
+ run,
+}: TransformGraphRunInfoSectionProps) {
+ const durationMs = getRunDurationMs(run);
+
+ return (
+ <Card p={0} shadow="none" withBorder role="region" aria-label={t`Info`}>
+ <SidebarInfoRow label={t`Type`}>{formatRunType(run)}</SidebarInfoRow>
+ <SidebarInfoRow label={t`Status`}>
+ <Box c={isErrorStatus(run.status) ? "feedback-negative" : undefined}>
+ {formatStatus(run.status)}
+ </Box>
+ </SidebarInfoRow>
+ <SidebarInfoRow label={t`Trigger`}>
+ {run.run_method != null
+ ? formatRunMethod(run.run_method)
+ : EMPTY_CELL_PLACEHOLDER}
+ </SidebarInfoRow>
+ <SidebarInfoRow label={t`Started at`}>
+ <DateTime value={run.start_time} unit="minute" />
+ </SidebarInfoRow>
+ {run.end_time != null && (
+ <SidebarInfoRow label={t`Ended at`}>
+ <DateTime value={run.end_time} unit="minute" />
+ </SidebarInfoRow>
+ )}
+ <SidebarInfoRow label={t`Duration`}>
+ {durationMs != null
+ ? formatDurationLong(durationMs)
+ : EMPTY_CELL_PLACEHOLDER}
+ </SidebarInfoRow>
+ </Card>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.css b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.css
new file mode 100644
index 00000000000..ce06f4b9e89
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.module.css
@@ -0,0 +1,12 @@
+.sidebar {
+ border-left: 1px solid var(--mb-color-border-neutral);
+}
+
+.header {
+ flex-shrink: 0;
+ border-bottom: 1px solid var(--mb-color-border-neutral);
+}
+
+.scrollArea {
+ overflow-y: auto;
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsx
new file mode 100644
index 00000000000..50bcd198fef
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/TransformGraphRunSidebar.tsx
@@ -0,0 +1,342 @@
+import { useDisclosure } from "@mantine/hooks";
+import { memo, useMemo, useState } from "react";
+import { match } from "ts-pattern";
+import { t } from "ttag";
+
+import {
+ skipToken,
+ useCancelCurrentTransformRunMutation,
+ useCancelDagRunMutation,
+ useCancelJobRunMutation,
+ useListDagRunTransformRunsQuery,
+ useListJobRunTransformRunsQuery,
+} from "metabase/api";
+import { ConfirmModal } from "metabase/common/components/ConfirmModal";
+import { ForwardRefLink } from "metabase/common/components/Link";
+import { ListEmptyState } from "metabase/common/components/ListEmptyState";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { PLUGIN_DEPENDENCIES } from "metabase/plugins";
+import { SidebarResizableBox } from "metabase/transforms/components/SidebarResizableBox";
+import { POLLING_INTERVAL } from "metabase/transforms/constants";
+import { isActiveRunStatus } from "metabase/transforms/utils";
+import {
+ ActionIcon,
+ Badge,
+ Box,
+ Button,
+ Card,
+ FixedSizeIcon,
+ Flex,
+ Group,
+ Stack,
+ Title,
+ Tooltip,
+} from "metabase/ui";
+import * as Urls from "metabase/urls";
+import { isResourceNotFoundError } from "metabase/utils/errors";
+import type {
+ TransformGraphRun,
+ TransformRunForJobRun,
+} from "metabase-types/api";
+
+import { TransformRunItem } from "../../JobRunListPage/JobRunSidebar/TransformRunItem";
+import { RunName, isDeletedRun } from "../TransformGraphRunTable";
+
+import { TransformGraphRunInfoSection } from "./TransformGraphRunInfoSection";
+import S from "./TransformGraphRunSidebar.module.css";
+
+const EMPTY_TRANSFORM_RUNS: TransformRunForJobRun[] = [];
+
+function canCancelRun(run: TransformGraphRun): boolean {
+ if (run.status !== "started") {
+ return false;
+ }
+ return run.run_type === "dag" || run.entity_id != null;
+}
+
+type TransformGraphRunSidebarProps = {
+ run: TransformGraphRun;
+ containerWidth: number;
+ onResizeStart: () => void;
+ onResizeStop: () => void;
+ onClose: () => void;
+};
+
+export const TransformGraphRunSidebar = memo(function TransformGraphRunSidebar({
+ run,
+ containerWidth,
+ onResizeStart,
+ onResizeStop,
+ onClose,
+}: TransformGraphRunSidebarProps) {
+ const [isPolling, setIsPolling] = useState(false);
+ const pollingInterval = isPolling ? POLLING_INTERVAL : undefined;
+
+ const areMembersUnavailable = run.run_type === "job" && isDeletedRun(run);
+
+ const dagResult = useListDagRunTransformRunsQuery(
+ run.run_type === "dag" ? { dagRunId: run.id } : skipToken,
+ { pollingInterval },
+ );
+ const jobResult = useListJobRunTransformRunsQuery(
+ run.run_type === "job" && run.entity_id != null
+ ? { jobId: run.entity_id, runId: run.id }
+ : skipToken,
+ { pollingInterval },
+ );
+ const standaloneRuns = useMemo<TransformRunForJobRun[]>(
+ () =>
+ run.run_type === "transform"
+ ? [
+ {
+ id: run.id,
+ transform_id: run.entity_id,
+ transform_name: run.name,
+ job_run_id: null,
+ status: run.status,
+ run_method: run.run_method ?? "manual",
+ start_time: run.start_time,
+ end_time: run.end_time,
+ message: run.message,
+ },
+ ]
+ : EMPTY_TRANSFORM_RUNS,
+ [run],
+ );
+
+ const { transformRuns, isLoading, error } = match(run.run_type)
+ .with("dag", () => ({
+ transformRuns: dagResult.data ?? EMPTY_TRANSFORM_RUNS,
+ isLoading: dagResult.isLoading,
+ error: dagResult.error,
+ }))
+ .with("job", () => ({
+ transformRuns: jobResult.data ?? EMPTY_TRANSFORM_RUNS,
+ isLoading: jobResult.isLoading,
+ error: jobResult.error,
+ }))
+ .with("transform", () => ({
+ transformRuns: standaloneRuns,
+ isLoading: false,
+ error: undefined,
+ }))
+ .exhaustive();
+
+ const shouldPoll =
+ isActiveRunStatus(run.status) ||
+ transformRuns.some((transformRun) =>
+ isActiveRunStatus(transformRun.status),
+ );
+ if (isPolling !== shouldPoll) {
+ setIsPolling(shouldPoll);
+ }
+
+ return (
+ <SidebarResizableBox
+ containerWidth={containerWidth}
+ onResizeStart={onResizeStart}
+ onResizeStop={onResizeStop}
+ >
+ <Flex
+ className={S.sidebar}
+ direction="column"
+ flex={1}
+ h="100%"
+ bg="background_page-primary"
+ data-testid="transform-graph-run-sidebar"
+ >
+ <Box className={S.header} p="lg">
+ <TransformGraphRunSidebarHeader run={run} onClose={onClose} />
+ </Box>
+ <Box className={S.scrollArea} flex={1} mih={0} p="lg">
+ <Stack gap="xl">
+ <Stack gap="md">
+ <TransformGraphRunInfoSection run={run} />
+ {canCancelRun(run) && <CancelationSection run={run} />}
+ </Stack>
+ {!areMembersUnavailable && (
+ <Stack gap="md">
+ <Group gap="sm" wrap="nowrap">
+ <Badge variant="filled" bg="core-brand">
+ {transformRuns.length}
+ </Badge>
+ <Title order={5}>{t`Transforms`}</Title>
+ </Group>
+ <TransformRunList
+ transformRuns={transformRuns}
+ isLoading={isLoading}
+ error={error}
+ />
+ </Stack>
+ )}
+ </Stack>
+ </Box>
+ </Flex>
+ </SidebarResizableBox>
+ );
+});
+
+type CancelationSectionProps = {
+ run: TransformGraphRun;
+};
+
+function CancelationSection({ run }: CancelationSectionProps) {
+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();
+ const { sendErrorToast } = useMetadataToasts();
+ const [cancelDagRun, { isLoading: isCancelingDag }] =
+ useCancelDagRunMutation();
+ const [cancelJobRun, { isLoading: isCancelingJob }] =
+ useCancelJobRunMutation();
+ const [cancelTransformRun, { isLoading: isCancelingTransform }] =
+ useCancelCurrentTransformRunMutation();
+ const isCanceling = isCancelingDag || isCancelingJob || isCancelingTransform;
+
+ const handleCancel = async () => {
+ let error: unknown;
+ if (run.run_type === "dag") {
+ ({ error } = await cancelDagRun(run.id));
+ } else if (run.run_type === "job" && run.entity_id != null) {
+ ({ error } = await cancelJobRun({ jobId: run.entity_id, runId: run.id }));
+ } else if (run.run_type === "transform" && run.entity_id != null) {
+ ({ error } = await cancelTransformRun(run.entity_id));
+ }
+ closeModal();
+ if (error && !isResourceNotFoundError(error)) {
+ sendErrorToast(t`Failed to cancel run`);
+ }
+ };
+
+ return (
+ <>
+ <Group justify="flex-end">
+ <Button
+ variant="filled"
+ color="feedback-negative"
+ disabled={isCanceling}
+ onClick={openModal}
+ >
+ {t`Cancel run`}
+ </Button>
+ </Group>
+ <ConfirmModal
+ title={t`Cancel this run?`}
+ message={t`This stops the run and requests cancellation of any transforms still in progress. Transforms that have already finished won't be reverted.`}
+ confirmButtonText={t`Cancel run`}
+ closeButtonText={t`Keep running`}
+ opened={isModalOpen}
+ onClose={closeModal}
+ onConfirm={handleCancel}
+ />
+ </>
+ );
+}
+
+type TransformRunListProps = {
+ transformRuns: TransformRunForJobRun[];
+ isLoading: boolean;
+ error: unknown;
+};
+
+function TransformRunList({
+ transformRuns,
+ isLoading,
+ error,
+}: TransformRunListProps) {
+ if (isLoading || error != null) {
+ return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
+ }
+
+ if (transformRuns.length === 0) {
+ return <ListEmptyState label={t`No transform runs`} />;
+ }
+
+ return (
+ <Card p={0} shadow="none" withBorder>
+ {transformRuns.map((transformRun) => (
+ <TransformRunItem key={transformRun.id} transformRun={transformRun} />
+ ))}
+ </Card>
+ );
+}
+
+type TransformGraphRunSidebarHeaderProps = {
+ run: TransformGraphRun;
+ onClose: () => void;
+};
+
+function TransformGraphRunSidebarHeader({
+ run,
+ onClose,
+}: TransformGraphRunSidebarHeaderProps) {
+ return (
+ <Group
+ justify="space-between"
+ align="center"
+ wrap="nowrap"
+ gap="sm"
+ data-testid="transform-graph-run-sidebar-header"
+ >
+ <Title order={3}>
+ <RunName run={run} gap="sm" />
+ </Title>
+ <Group gap="xs" wrap="nowrap">
+ <HeaderEntityActions run={run} />
+ <ActionIcon aria-label={t`Close`} onClick={onClose}>
+ <FixedSizeIcon name="close" />
+ </ActionIcon>
+ </Group>
+ </Group>
+ );
+}
+
+function HeaderEntityActions({ run }: { run: TransformGraphRun }) {
+ const { entity_id } = run;
+ if (entity_id == null) {
+ return null;
+ }
+
+ if (run.run_type === "job") {
+ return (
+ <Tooltip label={t`View this job`}>
+ <ActionIcon
+ component={ForwardRefLink}
+ to={Urls.transformJob(entity_id)}
+ target="_blank"
+ aria-label={t`View this job`}
+ >
+ <FixedSizeIcon name="external" />
+ </ActionIcon>
+ </Tooltip>
+ );
+ }
+
+ return (
+ <>
+ <Tooltip label={t`View this transform`}>
+ <ActionIcon
+ component={ForwardRefLink}
+ to={Urls.transform(entity_id)}
+ target="_blank"
+ aria-label={t`View this transform`}
+ >
+ <FixedSizeIcon name="external" />
+ </ActionIcon>
+ </Tooltip>
+ {PLUGIN_DEPENDENCIES.isEnabled && (
+ <Tooltip label={t`View in dependency graph`}>
+ <ActionIcon
+ component={ForwardRefLink}
+ to={Urls.dependencyGraph({
+ entry: { id: entity_id, type: "transform" },
+ })}
+ target="_blank"
+ aria-label={t`View in dependency graph`}
+ >
+ <FixedSizeIcon name="dependencies" />
+ </ActionIcon>
+ </Tooltip>
+ )}
+ </>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.ts
new file mode 100644
index 00000000000..5b56255155c
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunSidebar/index.ts
@@ -0,0 +1 @@
+export { TransformGraphRunSidebar } from "./TransformGraphRunSidebar";
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsx
new file mode 100644
index 00000000000..be317d53ca0
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/TransformGraphRunTable.tsx
@@ -0,0 +1,89 @@
+import type { Row, SortingState, Updater } from "@tanstack/react-table";
+import { useCallback, useMemo } from "react";
+import { t } from "ttag";
+
+import { ListEmptyState } from "metabase/common/components/ListEmptyState";
+import { useSetting } from "metabase/common/hooks";
+import { Card, TreeTable, useTreeTableInstance } from "metabase/ui";
+import type { TransformGraphRun } from "metabase-types/api";
+
+import type { TransformGraphRunSortOptions } from "../types";
+
+import {
+ getColumns,
+ getRowKey,
+ getSortingOptions,
+ getSortingState,
+} from "./utils";
+
+type TransformGraphRunTableProps = {
+ runs: TransformGraphRun[];
+ hasFilters: boolean;
+ sortOptions: TransformGraphRunSortOptions | undefined;
+ onSortOptionsChange: (
+ sortOptions: TransformGraphRunSortOptions | undefined,
+ ) => void;
+ onSelect: (run: TransformGraphRun) => void;
+};
+
+export function TransformGraphRunTable({
+ runs,
+ hasFilters,
+ sortOptions,
+ onSortOptionsChange,
+ onSelect,
+}: TransformGraphRunTableProps) {
+ const systemTimezone = useSetting("system-timezone");
+
+ const columns = useMemo(() => getColumns(systemTimezone), [systemTimezone]);
+
+ const sortingState = useMemo(
+ () => getSortingState(sortOptions),
+ [sortOptions],
+ );
+
+ const notFoundLabel = hasFilters ? t`No runs found` : t`No runs yet`;
+
+ const handleRowClick = useCallback(
+ (row: Row<TransformGraphRun>) => {
+ onSelect(row.original);
+ },
+ [onSelect],
+ );
+
+ const handleSortingChange = useCallback(
+ (updater: Updater<SortingState>) => {
+ const newSortingState =
+ typeof updater === "function" ? updater(sortingState) : updater;
+ onSortOptionsChange(getSortingOptions(newSortingState));
+ },
+ [sortingState, onSortOptionsChange],
+ );
+
+ const treeTableInstance = useTreeTableInstance<TransformGraphRun>({
+ data: runs,
+ columns,
+ sorting: sortingState,
+ manualSorting: true,
+ getNodeId: getRowKey,
+ onSortingChange: handleSortingChange,
+ });
+
+ return (
+ <Card
+ style={{ overflow: "hidden" }}
+ p={0}
+ flex="0 1 auto"
+ mih={0}
+ shadow="none"
+ withBorder
+ >
+ <TreeTable
+ instance={treeTableInstance}
+ emptyState={<ListEmptyState label={notFoundLabel} />}
+ ariaLabel={t`Runs`}
+ onRowClick={handleRowClick}
+ />
+ </Card>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.ts
new file mode 100644
index 00000000000..86ff3029a81
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/index.ts
@@ -0,0 +1,8 @@
+export { TransformGraphRunTable } from "./TransformGraphRunTable";
+export {
+ getRowKey,
+ formatRunName,
+ RunName,
+ formatRunType,
+ isDeletedRun,
+} from "./utils";
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsx b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsx
new file mode 100644
index 00000000000..c6e2407db2b
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/TransformGraphRunTable/utils.tsx
@@ -0,0 +1,306 @@
+import type { SortingState } from "@tanstack/react-table";
+import { msgid, ngettext, t } from "ttag";
+
+import { TimezoneIndicator } from "metabase/transforms/components/TimezoneIndicator";
+import {
+ formatRunMethod,
+ formatStatus,
+ getRunDurationMs,
+ isErrorStatus,
+ parseTimestampWithTimezone,
+} from "metabase/transforms/utils";
+import type { MantineSize, TreeTableColumnDef } from "metabase/ui";
+import {
+ Box,
+ Ellipsified,
+ Flex,
+ Group,
+ Icon,
+ SortableHeaderPill,
+ Text,
+ Tooltip,
+} from "metabase/ui";
+import { EMPTY_CELL_PLACEHOLDER } from "metabase/utils/constants";
+import { formatDurationLong } from "metabase/utils/formatting/time";
+import {
+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,
+ type TransformGraphRun,
+ type TransformGraphRunSortColumn,
+} from "metabase-types/api";
+
+import type { TransformGraphRunSortOptions } from "../types";
+
+export function getRowKey(run: TransformGraphRun): string {
+ return `${run.run_type}-${run.id}`;
+}
+
+export function isDeletedRun(run: TransformGraphRun): boolean {
+ return run.entity_id == null;
+}
+
+function dagDependencyCount(run: TransformGraphRun): number | null {
+ if (run.run_type !== "dag" || run.transform_count == null) {
+ return null;
+ }
+ const count = run.transform_count - 1; // Doesn't include transform that triggered this DAG run
+ return count > 0 ? count : null;
+}
+
+function dependenciesText(count: number): string {
+ return ngettext(msgid`${count} dependency`, `${count} dependencies`, count);
+}
+
+function dependentsText(count: number): string {
+ return ngettext(msgid`${count} dependent`, `${count} dependents`, count);
+}
+
+export function formatRunName(run: TransformGraphRun): string {
+ const name = run.name ?? t`Deleted`;
+ const count = dagDependencyCount(run);
+ if (count == null) {
+ return name;
+ }
+ return run.direction === "upstream"
+ ? `${dependenciesText(count)} › ${name}`
+ : `${name} › ${dependentsText(count)}`;
+}
+
+function RunEntityName({ run }: { run: TransformGraphRun }) {
+ const name = run.name ?? t`Deleted`;
+ if (!isDeletedRun(run)) {
+ return <Ellipsified fw="bold">{name}</Ellipsified>;
+ }
+ return (
+ <Ellipsified>
+ <Tooltip label={t`${name} has been deleted`}>
+ <Text c="text-disabled" component="span" display="inline" fs="italic">
+ {name}
+ </Text>
+ </Tooltip>
+ </Ellipsified>
+ );
+}
+
+type RunNameProps = {
+ run: TransformGraphRun;
+ gap?: MantineSize;
+};
+
+export function RunName({ run, gap = "xs" }: RunNameProps) {
+ const count = dagDependencyCount(run);
+ if (count == null) {
+ return <RunEntityName run={run} />;
+ }
+ const chevron = (
+ <Icon
+ name="chevronright"
+ c="icon-secondary"
+ size={8}
+ mt="0.15em"
+ aria-hidden
+ />
+ );
+ const countText = (
+ <Text
+ component="span"
+ c="text-secondary"
+ fz="inherit"
+ lh="inherit"
+ fw="normal"
+ style={{ whiteSpace: "nowrap" }}
+ >
+ {run.direction === "upstream"
+ ? dependenciesText(count)
+ : dependentsText(count)}
+ </Text>
+ );
+ return (
+ <Flex
+ component="span"
+ display="inline-flex"
+ align="center"
+ gap={gap}
+ miw={0}
+ >
+ {run.direction === "upstream" ? (
+ <>
+ {countText}
+ {chevron}
+ <RunEntityName run={run} />
+ </>
+ ) : (
+ <>
+ <RunEntityName run={run} />
+ {chevron}
+ {countText}
+ </>
+ )}
+ </Flex>
+ );
+}
+
+export function formatRunType(run: TransformGraphRun): string {
+ return run.run_type === "job" ? t`Job` : t`Transformation`;
+}
+
+function getRunColumn(): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "run",
+ header: t`Run`,
+ width: 320,
+ accessorFn: (run) => formatRunName(run),
+ cell: ({ row }) => <RunName run={row.original} />,
+ };
+}
+
+function getTypeColumn(): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "type",
+ header: t`Type`,
+ width: 140,
+ accessorFn: (run) => formatRunType(run),
+ cell: ({ getValue }) => <Ellipsified>{String(getValue())}</Ellipsified>,
+ };
+}
+
+function getStatusColumn(): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "status",
+ header: t`Status`,
+ width: 120,
+ accessorFn: (row) => formatStatus(row.status),
+ cell: ({ row }) => {
+ const { status } = row.original;
+ return (
+ <Box c={isErrorStatus(status) ? "feedback-negative" : undefined}>
+ {formatStatus(status)}
+ </Box>
+ );
+ },
+ };
+}
+
+function getRunMethodColumn(): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "run-method",
+ header: t`Trigger`,
+ width: 120,
+ accessorFn: (row) =>
+ row.run_method != null ? formatRunMethod(row.run_method) : null,
+ cell: ({ getValue }) => {
+ const value = getValue<string | null>();
+ if (value == null) {
+ return EMPTY_CELL_PLACEHOLDER;
+ }
+ return <Ellipsified>{value}</Ellipsified>;
+ },
+ };
+}
+
+function getStartedAtColumn(
+ systemTimezone: string | undefined,
+): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "start_time" satisfies TransformGraphRunSortColumn,
+ header: ({ header }) => (
+ <Group gap="xs" wrap="nowrap">
+ <SortableHeaderPill
+ name={t`Started at`}
+ sort={header.column.getIsSorted() || undefined}
+ />
+ <TimezoneIndicator />
+ </Group>
+ ),
+ width: "auto",
+ enableSorting: true,
+ accessorFn: (run) =>
+ parseTimestampWithTimezone(run.start_time, systemTimezone).format("lll"),
+ cell: ({ getValue }) => <Ellipsified>{String(getValue())}</Ellipsified>,
+ };
+}
+
+function getEndedAtColumn(
+ systemTimezone: string | undefined,
+): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "end_time" satisfies TransformGraphRunSortColumn,
+ header: ({ header }) => (
+ <Group gap="xs" wrap="nowrap">
+ <SortableHeaderPill
+ name={t`Ended at`}
+ sort={header.column.getIsSorted() || undefined}
+ />
+ <TimezoneIndicator />
+ </Group>
+ ),
+ width: "auto",
+ enableSorting: true,
+ accessorFn: (run) =>
+ run.end_time != null
+ ? parseTimestampWithTimezone(run.end_time, systemTimezone).format("lll")
+ : null,
+ cell: ({ getValue }) => {
+ const value = getValue<string | null>();
+ if (value == null) {
+ return EMPTY_CELL_PLACEHOLDER;
+ }
+ return <Ellipsified>{value}</Ellipsified>;
+ },
+ };
+}
+
+function getDurationColumn(): TreeTableColumnDef<TransformGraphRun> {
+ return {
+ id: "duration",
+ header: t`Duration`,
+ width: 120,
+ accessorFn: (run) => getRunDurationMs(run),
+ cell: ({ getValue }) => {
+ const ms = getValue<number | null>();
+ if (ms == null) {
+ return EMPTY_CELL_PLACEHOLDER;
+ }
+ return <Ellipsified>{formatDurationLong(ms)}</Ellipsified>;
+ },
+ };
+}
+
+export function getColumns(
+ systemTimezone: string | undefined,
+): TreeTableColumnDef<TransformGraphRun>[] {
+ return [
+ getRunColumn(),
+ getTypeColumn(),
+ getStartedAtColumn(systemTimezone),
+ getEndedAtColumn(systemTimezone),
+ getDurationColumn(),
+ getStatusColumn(),
+ getRunMethodColumn(),
+ ];
+}
+
+export function getSortingState(
+ sortOptions: TransformGraphRunSortOptions | undefined,
+): SortingState {
+ return sortOptions != null
+ ? [{ id: sortOptions.column, desc: sortOptions.direction === "desc" }]
+ : [];
+}
+
+export function getSortingOptions(
+ sortingState: SortingState,
+): TransformGraphRunSortOptions | undefined {
+ if (sortingState.length === 0) {
+ return undefined;
+ }
+
+ const { id, desc } = sortingState[0];
+ const column = TRANSFORM_GRAPH_RUN_SORT_COLUMNS.find(
+ (column) => column === id,
+ );
+ if (column == null) {
+ return undefined;
+ }
+
+ return { column, direction: desc ? "desc" : "asc" };
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.ts
new file mode 100644
index 00000000000..45d559e788d
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/constants.ts
@@ -0,0 +1 @@
+export const PAGE_SIZE = 25;
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.ts
new file mode 100644
index 00000000000..4e598a5e24a
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/index.ts
@@ -0,0 +1 @@
+export { TransformGraphRunListPage } from "./TransformGraphRunListPage";
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.ts
new file mode 100644
index 00000000000..1960bb6aa5c
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/types.ts
@@ -0,0 +1,22 @@
+import type {
+ SortDirection,
+ TransformGraphRunSortColumn,
+ TransformGraphRunType,
+ TransformId,
+ TransformRunMethod,
+ TransformRunStatus,
+} from "metabase-types/api";
+
+export type TransformGraphRunFilterOptions = {
+ types?: TransformGraphRunType[];
+ statuses?: TransformRunStatus[];
+ transformIds?: TransformId[];
+ startTime?: string;
+ endTime?: string;
+ runMethods?: TransformRunMethod[];
+};
+
+export type TransformGraphRunSortOptions = {
+ column: TransformGraphRunSortColumn;
+ direction: SortDirection;
+};
diff --git a/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.ts
new file mode 100644
index 00000000000..70de18c9d42
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformGraphRunListPage/utils.ts
@@ -0,0 +1,86 @@
+import type { Location } from "history";
+
+import * as Urls from "metabase/urls";
+import {
+ SORT_DIRECTIONS,
+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,
+ TRANSFORM_GRAPH_RUN_TYPES,
+ TRANSFORM_RUN_METHODS,
+ TRANSFORM_RUN_STATUSES,
+} from "metabase-types/api";
+
+import type {
+ TransformGraphRunFilterOptions,
+ TransformGraphRunSortOptions,
+} from "./types";
+
+export function getParsedParams(
+ location: Location,
+): Urls.TransformGraphRunListParams {
+ const {
+ page,
+ types,
+ statuses,
+ "transform-ids": transformIds,
+ "start-time": startTime,
+ "end-time": endTime,
+ "run-methods": runMethods,
+ "sort-column": sortColumn,
+ "sort-direction": sortDirection,
+ } = location.query;
+
+ return {
+ page: Urls.parseNumberParam(page),
+ types: Urls.parseListParam(types, (v) =>
+ Urls.parseEnumParam(v, TRANSFORM_GRAPH_RUN_TYPES),
+ ),
+ statuses: Urls.parseListParam(statuses, (v) =>
+ Urls.parseEnumParam(v, TRANSFORM_RUN_STATUSES),
+ ),
+ transformIds: Urls.parseListParam(transformIds, Urls.parseNumberParam),
+ startTime: Urls.parseStringParam(startTime),
+ endTime: Urls.parseStringParam(endTime),
+ runMethods: Urls.parseListParam(runMethods, (v) =>
+ Urls.parseEnumParam(v, TRANSFORM_RUN_METHODS),
+ ),
+ sortColumn: Urls.parseEnumParam(
+ sortColumn,
+ TRANSFORM_GRAPH_RUN_SORT_COLUMNS,
+ ),
+ sortDirection: Urls.parseEnumParam(sortDirection, SORT_DIRECTIONS),
+ };
+}
+
+export function getFilterOptions(
+ params: Urls.TransformGraphRunListParams,
+): TransformGraphRunFilterOptions {
+ return {
+ types: params.types,
+ statuses: params.statuses,
+ transformIds: params.transformIds,
+ startTime: params.startTime,
+ endTime: params.endTime,
+ runMethods: params.runMethods,
+ };
+}
+
+export function hasFilterOptions(
+ options: TransformGraphRunFilterOptions,
+): boolean {
+ return (
+ options.types != null ||
+ options.statuses != null ||
+ options.transformIds != null ||
+ options.startTime != null ||
+ options.endTime != null ||
+ options.runMethods != null
+ );
+}
+
+export function getSortOptions(
+ params: Urls.TransformGraphRunListParams,
+): TransformGraphRunSortOptions | undefined {
+ return params.sortColumn != null && params.sortDirection != null
+ ? { column: params.sortColumn, direction: params.sortDirection }
+ : undefined;
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsx
new file mode 100644
index 00000000000..c75b7e5beaa
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunDagConfirmModal.tsx
@@ -0,0 +1,76 @@
+import { t } from "ttag";
+
+import { skipToken, useListDagTransformsQuery } from "metabase/api";
+import {
+ Button,
+ Center,
+ Group,
+ List,
+ Loader,
+ Modal,
+ ScrollArea,
+ Stack,
+ Text,
+} from "metabase/ui";
+import type { TransformDagDirection, TransformId } from "metabase-types/api";
+
+type RunDagConfirmModalProps = {
+ transformId: TransformId;
+ direction: TransformDagDirection | null;
+ isConfirming?: boolean;
+ onClose: () => void;
+ onConfirm: () => void;
+};
+
+export function RunDagConfirmModal({
+ transformId,
+ direction,
+ isConfirming,
+ onClose,
+ onConfirm,
+}: RunDagConfirmModalProps) {
+ const { data: transforms, isFetching } = useListDagTransformsQuery(
+ direction ? { transformId, direction } : skipToken,
+ );
+
+ const title =
+ direction === "upstream"
+ ? t`Run this and all upstream transforms?`
+ : t`Run this and all downstream transforms?`;
+
+ return (
+ <Modal
+ opened={direction != null}
+ title={title}
+ padding="xl"
+ onClose={onClose}
+ >
+ <Stack gap="lg">
+ <Text>{t`These transforms will be run:`}</Text>
+ {isFetching ? (
+ <Center py="lg">
+ <Loader />
+ </Center>
+ ) : (
+ <ScrollArea.Autosize mah="50vh">
+ <List>
+ {(transforms ?? []).map((transform) => (
+ <List.Item key={transform.id}>{transform.name}</List.Item>
+ ))}
+ </List>
+ </ScrollArea.Autosize>
+ )}
+ <Group justify="flex-end">
+ <Button onClick={onClose}>{t`Cancel`}</Button>
+ <Button
+ variant="filled"
+ disabled={isFetching || isConfirming}
+ onClick={onConfirm}
+ >
+ {t`Run all`}
+ </Button>
+ </Group>
+ </Stack>
+ </Modal>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
index cda55ac8cd6..48a8047251d 100644
--- a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
@@ -1,9 +1,13 @@
import { useDisclosure } from "@mantine/hooks";
+import { useCallback, useEffect, useState } from "react";
import { usePrevious } from "react-use";
import { t } from "ttag";
import {
+ skipToken,
useCancelCurrentTransformRunMutation,
+ useListDagRunTransformRunsQuery,
+ useRunTransformDagMutation,
useRunTransformMutation,
useUpdateTransformMutation,
} from "metabase/api";
@@ -13,13 +17,30 @@ import { useMetadataToasts } from "metabase/metadata/hooks";
import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";
import { useSelector } from "metabase/redux";
import { Link } from "metabase/router";
-import { Anchor, Box, Card, Divider, Group, Stack } from "metabase/ui";
+import { POLLING_INTERVAL } from "metabase/transforms/constants";
+import { isActiveRunStatus } from "metabase/transforms/utils";
+import {
+ Anchor,
+ Box,
+ Card,
+ Divider,
+ Group,
+ Icon,
+ Menu,
+ Stack,
+} from "metabase/ui";
import * as Urls from "metabase/urls";
import { isResourceNotFoundError } from "metabase/utils/errors";
-import type { Transform, TransformTagId } from "metabase-types/api";
+import type {
+ Transform,
+ TransformDagDirection,
+ TransformDagRunId,
+ TransformTagId,
+} from "metabase-types/api";
import {
trackTransformRunTagsUpdated,
+ trackTransformTriggerDagRun,
trackTransformTriggerManualRun,
} from "../../../analytics";
import { RunButton } from "../../../components/RunButton";
@@ -27,6 +48,7 @@ import { RunStatus } from "../../../components/RunStatus";
import { TagMultiSelect } from "../../../components/TagMultiSelect";
import { LogOutput } from "./LogOutput";
+import { RunDagConfirmModal } from "./RunDagConfirmModal";
type RunSectionProps = {
transform: Transform;
@@ -34,17 +56,65 @@ type RunSectionProps = {
noTitle?: boolean;
};
+function useScheduledDagRun(transform: Transform) {
+ const [scheduledDagRunId, setScheduledDagRunId] =
+ useState<TransformDagRunId | null>(null);
+
+ const { data: members, error } = useListDagRunTransformRunsQuery(
+ scheduledDagRunId != null ? { dagRunId: scheduledDagRunId } : skipToken,
+ {
+ pollingInterval: scheduledDagRunId != null ? POLLING_INTERVAL : undefined,
+ },
+ );
+
+ const dagFinished =
+ members != null &&
+ members.length > 0 &&
+ !members.some((run) => isActiveRunStatus(run.status));
+
+ const dagUnavailable = error != null;
+
+ useEffect(() => {
+ if (scheduledDagRunId != null && (dagFinished || dagUnavailable)) {
+ setScheduledDagRunId(null);
+ }
+ }, [scheduledDagRunId, dagFinished, dagUnavailable]);
+
+ const hasRunThisCycle = members?.some(
+ (run) => run.transform_id === transform.id,
+ );
+
+ const schedule = useCallback(
+ (dagRunId: TransformDagRunId) => setScheduledDagRunId(dagRunId),
+ [],
+ );
+
+ return {
+ isScheduled:
+ scheduledDagRunId != null &&
+ !dagFinished &&
+ !dagUnavailable &&
+ !hasRunThisCycle,
+ schedule,
+ };
+}
+
export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {
const isRemoteSyncReadOnly = useSelector(
PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,
);
+ const { isScheduled, schedule } = useScheduledDagRun(transform);
const content = (
<>
<Stack>
<Group p="lg" justify="space-between">
- <RunStatusSection transform={transform} />
- <RunButtonSection transform={transform} readOnly={readOnly} />
+ <RunStatusSection transform={transform} isScheduled={isScheduled} />
+ <RunButtonSection
+ transform={transform}
+ readOnly={readOnly}
+ onScheduled={schedule}
+ />
</Group>
<RunOutputSection transform={transform} />
</Stack>
@@ -82,9 +152,10 @@ export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {
type RunStatusSectionProps = {
transform: Transform;
+ isScheduled: boolean;
};
-function RunStatusSection({ transform }: RunStatusSectionProps) {
+function RunStatusSection({ transform, isScheduled }: RunStatusSectionProps) {
const { id, last_run } = transform;
const status = last_run?.status;
@@ -97,6 +168,15 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {
>{t`This run succeeded before it had a chance to cancel.`}</Box>
);
+ if (isScheduled) {
+ return (
+ <Group gap="sm" data-testid="run-status">
+ <Icon c="text-secondary" name="clock" />
+ <Box>{t`Scheduled to run as part of a reprocess run.`}</Box>
+ </Group>
+ );
+ }
+
return (
<Stack gap={0}>
<RunStatus
@@ -107,6 +187,7 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {
key="link"
component={Link}
to={Urls.transformRunList({ transformIds: [id] })}
+ lh="inherit"
>
{t`See all runs`}
</Anchor>
@@ -120,16 +201,25 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {
type RunButtonSectionProps = {
transform: Transform;
readOnly?: boolean;
+ onScheduled: (dagRunId: TransformDagRunId) => void;
};
-function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {
+function RunButtonSection({
+ transform,
+ readOnly,
+ onScheduled,
+}: RunButtonSectionProps) {
const [runTransform] = useRunTransformMutation();
+ const [runTransformDag, { isLoading: isSubmittingDag }] =
+ useRunTransformDagMutation();
const [cancelTransform] = useCancelCurrentTransformRunMutation();
- const { sendErrorToast } = useMetadataToasts();
+ const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const [
isConfirmCancellationModalOpen,
{ close: closeConfirmModal, open: openConfirmModal },
] = useDisclosure(false);
+ const [dagDirection, setDagDirection] =
+ useState<TransformDagDirection | null>(null);
const handleRun = async () => {
trackTransformTriggerManualRun({ transformId: transform.id });
@@ -139,6 +229,30 @@ function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {
}
};
+ const handleRunDag = async () => {
+ const direction = dagDirection;
+ if (direction == null) {
+ return;
+ }
+ trackTransformTriggerDagRun({ transformId: transform.id, direction });
+ const { data, error } = await runTransformDag({
+ id: transform.id,
+ direction,
+ });
+ setDagDirection(null);
+ if (error) {
+ sendErrorToast(t`Failed to run transforms`);
+ return;
+ }
+ if (data?.dag_run_id != null) {
+ onScheduled(data.dag_run_id);
+ } else {
+ sendSuccessToast(
+ t`A reprocess run for this transform is already in progress.`,
+ );
+ }
+ };
+
const handleCancel = async () => {
const { error } = await cancelTransform(transform.id);
if (error && !isResourceNotFoundError(error)) {
@@ -155,16 +269,37 @@ function RunButtonSection({ transform, readOnly }: RunButtonSectionProps) {
onRun={handleRun}
onCancel={openConfirmModal}
isDisabled={readOnly}
+ menuItems={
+ readOnly ? undefined : (
+ <>
+ <Menu.Item onClick={() => setDagDirection("upstream")}>
+ {t`Run this and all upstream transforms`}
+ </Menu.Item>
+ <Menu.Item onClick={() => setDagDirection("downstream")}>
+ {t`Run this and all downstream transforms`}
+ </Menu.Item>
+ </>
+ )
+ }
+ />
+ <RunDagConfirmModal
+ transformId={transform.id}
+ direction={dagDirection}
+ isConfirming={isSubmittingDag}
+ onClose={() => setDagDirection(null)}
+ onConfirm={handleRunDag}
/>
<ConfirmModal
title={t`Cancel this run?`}
+ message={t`This requests the run to stop; it may take a moment to finish canceling.`}
+ confirmButtonText={t`Cancel run`}
+ closeButtonText={t`Keep running`}
opened={isConfirmCancellationModalOpen}
onClose={closeConfirmModal}
onConfirm={() => {
void handleCancel();
closeConfirmModal();
}}
- closeButtonText={t`No`}
/>
</>
);
diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsx
new file mode 100644
index 00000000000..79a18ef9569
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.unit.spec.tsx
@@ -0,0 +1,173 @@
+import userEvent from "@testing-library/user-event";
+import fetchMock from "fetch-mock";
+
+import {
+ setupListDagRunTransformRunsEndpoint,
+ setupListDagTransformsEndpoint,
+ setupListTransformTagsEndpoint,
+ setupRunTransformDagEndpoint,
+} from "__support__/server-mocks";
+import { renderWithProviders, screen, waitFor } from "__support__/ui";
+import { UndoListing } from "metabase/common/components/UndoListing";
+import type {
+ RunTransformDagResponse,
+ Transform,
+ TransformRunForJobRun,
+} from "metabase-types/api";
+import {
+ createMockTransform,
+ createMockTransformRun,
+ createMockTransformRunForJobRun,
+} from "metabase-types/api/mocks";
+
+import { RunSection } from "./RunSection";
+
+const TRANSFORM_ID = 10;
+const DAG_RUN_ID = 1;
+
+const DEFAULT_DAG_MEMBERS: TransformRunForJobRun[] = [
+ createMockTransformRunForJobRun({
+ id: 1,
+ transform_id: 2,
+ status: "started",
+ }),
+];
+
+function setup({
+ transform,
+ runDagResponse,
+ dagMembers = DEFAULT_DAG_MEMBERS,
+}: {
+ transform?: Transform;
+ runDagResponse?: RunTransformDagResponse;
+ dagMembers?: TransformRunForJobRun[];
+} = {}) {
+ const testTransform =
+ transform ?? createMockTransform({ id: TRANSFORM_ID, last_run: null });
+
+ setupListTransformTagsEndpoint([]);
+ setupListDagTransformsEndpoint(TRANSFORM_ID, [
+ { id: 1, name: "Upstream A" },
+ { id: 2, name: "Upstream B" },
+ { id: TRANSFORM_ID, name: testTransform.name },
+ ]);
+ setupRunTransformDagEndpoint(TRANSFORM_ID, runDagResponse);
+ setupListDagRunTransformRunsEndpoint(DAG_RUN_ID, dagMembers);
+
+ renderWithProviders(
+ <>
+ <RunSection transform={testTransform} />
+ <UndoListing />
+ </>,
+ );
+}
+
+async function triggerUpstreamRun() {
+ await userEvent.click(screen.getByTestId("run-options-button"));
+ await userEvent.click(
+ await screen.findByText("Run this and all upstream transforms"),
+ );
+ await screen.findByText("Upstream A");
+ await userEvent.click(screen.getByRole("button", { name: "Run all" }));
+}
+
+describe("RunSection DAG run flow", () => {
+ it("offers upstream and downstream run options", async () => {
+ setup();
+ await userEvent.click(screen.getByTestId("run-options-button"));
+
+ expect(
+ await screen.findByText("Run this and all upstream transforms"),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByText("Run this and all downstream transforms"),
+ ).toBeInTheDocument();
+ });
+
+ it("previews the transforms and triggers a run-dag with the chosen direction", async () => {
+ setup();
+ await userEvent.click(screen.getByTestId("run-options-button"));
+ await userEvent.click(
+ await screen.findByText("Run this and all upstream transforms"),
+ );
+
+ expect(
+ await screen.findByText("Run this and all upstream transforms?"),
+ ).toBeInTheDocument();
+ expect(await screen.findByText("Upstream A")).toBeInTheDocument();
+ expect(screen.getByText("Upstream B")).toBeInTheDocument();
+
+ await userEvent.click(screen.getByRole("button", { name: "Run all" }));
+
+ await waitFor(() => {
+ expect(
+ fetchMock.callHistory.called(
+ `path:/api/transform/${TRANSFORM_ID}/run-dag`,
+ ),
+ ).toBe(true);
+ });
+ const call = fetchMock.callHistory.lastCall(
+ `path:/api/transform/${TRANSFORM_ID}/run-dag`,
+ );
+ const rawBody = call?.options?.body;
+ const body = JSON.parse(typeof rawBody === "string" ? rawBody : "{}");
+ expect(body.direction).toBe("upstream");
+ });
+
+ it("shows a scheduled state after triggering a DAG run", async () => {
+ setup();
+ await triggerUpstreamRun();
+
+ expect(
+ await screen.findByText("Scheduled to run as part of a reprocess run."),
+ ).toBeInTheDocument();
+ });
+
+ it("stops showing the scheduled state once this transform's own run has completed mid-DAG", async () => {
+ setup({
+ transform: createMockTransform({
+ id: TRANSFORM_ID,
+ last_run: createMockTransformRun({
+ status: "succeeded",
+ end_time: null,
+ }),
+ }),
+ dagMembers: [
+ createMockTransformRunForJobRun({
+ id: 1,
+ transform_id: 2,
+ status: "started",
+ }),
+ createMockTransformRunForJobRun({
+ id: 2,
+ transform_id: TRANSFORM_ID,
+ status: "succeeded",
+ }),
+ ],
+ });
+ await triggerUpstreamRun();
+
+ expect(
+ await screen.findByText("Last ran successfully."),
+ ).toBeInTheDocument();
+ expect(
+ screen.queryByText("Scheduled to run as part of a reprocess run."),
+ ).not.toBeInTheDocument();
+ });
+
+ it("notifies the user and skips the scheduled state when nothing was run", async () => {
+ setup({
+ runDagResponse: { message: "DAG run started", dag_run_id: null },
+ });
+ await triggerUpstreamRun();
+
+ expect(
+ await screen.findByText(
+ "A reprocess run for this transform is already in progress.",
+ ),
+ ).toBeInTheDocument();
+ expect(
+ screen.queryByText("Scheduled to run as part of a reprocess run."),
+ ).not.toBeInTheDocument();
+ });
+});
diff --git a/frontend/src/metabase/transforms/routes.tsx b/frontend/src/metabase/transforms/routes.tsx
index bbef73f2d40..4bfd4e7a16b 100644
--- a/frontend/src/metabase/transforms/routes.tsx
+++ b/frontend/src/metabase/transforms/routes.tsx
@@ -16,7 +16,9 @@ import {
NewQueryTransformPage,
} from "./pages/NewTransformPage";
import { RunListPage } from "./pages/RunListPage";
+import { RunsPage } from "./pages/RunsPage";
import { TransformDependenciesPage } from "./pages/TransformDependenciesPage";
+import { TransformGraphRunListPage } from "./pages/TransformGraphRunListPage";
import { TransformIndexesPage } from "./pages/TransformIndexesPage";
import { TransformListPage } from "./pages/TransformListPage";
import { TransformQueryPage } from "./pages/TransformQueryPage";
@@ -26,6 +28,9 @@ import { TransformsNotDisabled } from "./route-guards";
const RoutedTransformListPage = withRouteProps(TransformListPage);
const RoutedRunListPage = withRouteProps(RunListPage);
+const RoutedTransformGraphRunListPage = withRouteProps(
+ TransformGraphRunListPage,
+);
const RoutedNewJobPage = withRouteProps(NewJobPage);
const RoutedJobPage = withRouteProps(JobPage);
const RoutedJobRunListPage = withRouteProps(JobRunListPage);
@@ -44,7 +49,10 @@ export function getDataStudioTransformRoutes() {
return (
<Route element={<TransformsNotDisabled />}>
<Route index element={<RoutedTransformListPage />} />
- <Route path="runs" element={<RoutedRunListPage />} />
+ <Route path="runs" element={<RunsPage />}>
+ <Route index element={<RoutedTransformGraphRunListPage />} />
+ <Route path="individual" element={<RoutedRunListPage />} />
+ </Route>
<Route path="jobs" element={<JobSectionLayout />}>
<Route index element={<JobListPage />} />
<Route path="new" element={<RoutedNewJobPage />} />
diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.ts
index 2a284aeb6ed..5c312ce795a 100644
--- a/frontend/src/metabase/transforms/utils.ts
+++ b/frontend/src/metabase/transforms/utils.ts
@@ -122,6 +122,12 @@ export function isErrorStatus(status: TransformRunStatus | null) {
return status === "failed" || status === "timeout";
}
+export function isActiveRunStatus(
+ status: TransformRunStatus | null | undefined,
+) {
+ return status === "started" || status === "canceling";
+}
+
export function isTransformRunning(transform: Transform) {
const lastRun = transform.last_run;
return lastRun?.status === "started";
diff --git a/frontend/src/metabase/urls/transforms.ts b/frontend/src/metabase/urls/transforms.ts
index a705c3afb0b..89d9ba62cbb 100644
--- a/frontend/src/metabase/urls/transforms.ts
+++ b/frontend/src/metabase/urls/transforms.ts
@@ -1,9 +1,13 @@
+import _ from "underscore";
+
import type {
CardId,
CollectionId,
DatabaseId,
SortDirection,
TableId,
+ TransformGraphRunSortColumn,
+ TransformGraphRunType,
TransformId,
TransformJobId,
TransformJobRunSortColumn,
@@ -17,6 +21,7 @@ import type {
const TRANSFORMS_ROOT_URL = `/data-studio/transforms`;
const JOBS_ROOT_URL = `${TRANSFORMS_ROOT_URL}/jobs`;
const RUNS_ROOT_URL = `${TRANSFORMS_ROOT_URL}/runs`;
+const INDIVIDUAL_RUNS_ROOT_URL = `${RUNS_ROOT_URL}/individual`;
const LIBRARY_ROOT_URL = `${TRANSFORMS_ROOT_URL}/library`;
export type TransformPythonLibraryParams = {
@@ -198,12 +203,91 @@ export function transformRunList({
const queryString = searchParams.toString();
if (queryString.length > 0) {
- return `${RUNS_ROOT_URL}?${queryString}`;
+ return `${INDIVIDUAL_RUNS_ROOT_URL}?${queryString}`;
} else {
- return RUNS_ROOT_URL;
+ return INDIVIDUAL_RUNS_ROOT_URL;
}
}
+export type TransformGraphRunListParams = {
+ page?: number;
+ types?: TransformGraphRunType[];
+ statuses?: TransformRunStatus[];
+ transformIds?: TransformId[];
+ startTime?: string;
+ endTime?: string;
+ runMethods?: TransformRunMethod[];
+ sortColumn?: TransformGraphRunSortColumn;
+ sortDirection?: SortDirection;
+};
+
+export function transformGraphRunList({
+ page,
+ types,
+ statuses,
+ transformIds,
+ startTime,
+ endTime,
+ runMethods,
+ sortColumn,
+ sortDirection,
+}: TransformGraphRunListParams = {}) {
+ const searchParams = new URLSearchParams();
+ if (page != null) {
+ searchParams.set("page", String(page));
+ }
+ types?.forEach((type) => {
+ searchParams.append("types", type);
+ });
+ statuses?.forEach((status) => {
+ searchParams.append("statuses", status);
+ });
+ transformIds?.forEach((transformId) => {
+ searchParams.append("transform-ids", String(transformId));
+ });
+ if (startTime != null) {
+ searchParams.set("start-time", startTime);
+ }
+ if (endTime != null) {
+ searchParams.set("end-time", endTime);
+ }
+ runMethods?.forEach((runMethod) => {
+ searchParams.append("run-methods", runMethod);
+ });
+ if (sortColumn != null) {
+ searchParams.set("sort-column", sortColumn);
+ }
+ if (sortDirection != null) {
+ searchParams.set("sort-direction", sortDirection);
+ }
+
+ const queryString = searchParams.toString();
+ return queryString.length > 0
+ ? `${RUNS_ROOT_URL}?${queryString}`
+ : RUNS_ROOT_URL;
+}
+
+export type CommonRunListParams = {
+ statuses?: TransformRunStatus[];
+ transformIds?: TransformId[];
+ startTime?: string;
+ endTime?: string;
+ runMethods?: TransformRunMethod[];
+};
+
+export function pickCommonRunListParams(
+ params: CommonRunListParams,
+): CommonRunListParams {
+ return _.pick(
+ params,
+ "statuses",
+ "transformIds",
+ "startTime",
+ "endTime",
+ "runMethods",
+ );
+}
+
export function transformPythonLibrary({ path }: TransformPythonLibraryParams) {
return `${LIBRARY_ROOT_URL}/${path}`;
}
diff --git a/frontend/test/__support__/server-mocks/transform.ts b/frontend/test/__support__/server-mocks/transform.ts
index b66c6891a15..3563d4e44f1 100644
--- a/frontend/test/__support__/server-mocks/transform.ts
+++ b/frontend/test/__support__/server-mocks/transform.ts
@@ -1,11 +1,15 @@
import fetchMock from "fetch-mock";
import type {
+ DagTransform,
Dataset,
InspectorLensId,
+ ListTransformGraphRunsResponse,
ListTransformJobRunsResponse,
ListTransformRunsResponse,
+ RunTransformDagResponse,
Transform,
+ TransformDagRunId,
TransformId,
TransformJob,
TransformJobId,
@@ -20,6 +24,12 @@ export function setupListTransformRunsEndpoint(
fetchMock.get(`path:/api/transform/run`, response);
}
+export function setupListTransformGraphRunsEndpoint(
+ response: ListTransformGraphRunsResponse,
+) {
+ fetchMock.get(`path:/api/transform/runs`, response);
+}
+
export function setupListTransformsEndpoint(transforms: Transform[]) {
fetchMock.get(`path:/api/transform`, transforms);
}
@@ -84,6 +94,40 @@ export function setupListJobRunTransformRunsEndpoint(
);
}
+export function setupListDagTransformsEndpoint(
+ transformId: TransformId,
+ transforms: DagTransform[],
+) {
+ fetchMock.get(
+ `path:/api/transform/${transformId}/dag-transforms`,
+ transforms,
+ );
+}
+
+export function setupRunTransformDagEndpoint(
+ transformId: TransformId,
+ response: RunTransformDagResponse = {
+ message: "DAG run started",
+ dag_run_id: 1,
+ },
+) {
+ fetchMock.post(`path:/api/transform/${transformId}/run-dag`, response);
+}
+
+export function setupListDagRunTransformRunsEndpoint(
+ dagRunId: TransformDagRunId,
+ runs: TransformRunForJobRun[] | (() => TransformRunForJobRun[]),
+) {
+ fetchMock.get(`path:/api/transform-dag-run/${dagRunId}/transform-runs`, runs);
+}
+
+export function setupCancelJobRunEndpoint(
+ jobId: TransformJobId,
+ runId: TransformJobRunId,
+) {
+ fetchMock.post(`path:/api/transform-job/${jobId}/runs/${runId}/cancel`, 204);
+}
+
export function setupCreateTransformJobEndpoint(job: TransformJob) {
fetchMock.post("path:/api/transform-job", job);
}
diff --git a/resources/migrations/064/20260703_transform_dag_run.yaml b/resources/migrations/064/20260703_transform_dag_run.yaml
new file mode 100644
index 00000000000..6393c205d02
--- /dev/null
+++ b/resources/migrations/064/20260703_transform_dag_run.yaml
@@ -0,0 +1,349 @@
+## DAG-reprocess runs: reprocessing the transitive dependency DAG rooted at a single transform
+## (upstream or downstream). These are coordinated multi-transform runs, like scheduled job runs,
+## but triggered manually from one seed transform rather than by a job. They get their own table
+## (transform_dag_run) rather than overloading transform_job_run, and each member transform_run is
+## linked back to its coordinating DAG run via transform_run.dag_run_id (mirroring job_run_id).
+
+databaseChangeLog:
+
+ - changeSet:
+ id: v64.oxRKX6
+ author: bronsa
+ comment: Create transform_dag_run table to track manual DAG-reprocess runs seeded from a single transform
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - tableExists:
+ tableName: transform_dag_run
+ changes:
+ - createTable:
+ tableName: transform_dag_run
+ remarks: Tracks manual DAG-reprocess runs (reprocessing a transform's upstream/downstream dependency DAG)
+ columns:
+ - column:
+ name: id
+ type: int
+ remarks: Unique ID
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_transform_id
+ type: int
+ remarks: The seed transform the DAG reprocess was triggered from (null if it was later deleted)
+ constraints:
+ nullable: true
+ - column:
+ name: source_transform_name
+ type: ${text.type}
+ remarks: Name of the seed transform at run start, preserved after its deletion
+ constraints:
+ nullable: true
+ - column:
+ name: source_transform_entity_id
+ type: char(21)
+ remarks: entity_id of the seed transform at run start, for potential reconnection when it is recreated
+ constraints:
+ nullable: true
+ - column:
+ name: direction
+ type: varchar(20)
+ remarks: DAG traversal direction (upstream/downstream) that selected the transforms to run
+ constraints:
+ nullable: false
+ - column:
+ name: transform_count
+ type: int
+ remarks: Number of transforms the run's plan selected to run, for display in run listings
+ constraints:
+ nullable: true
+ - column:
+ name: status
+ type: varchar(32)
+ remarks: Current status of the run (started, succeeded, failed, timeout, canceled)
+ constraints:
+ nullable: false
+ - column:
+ name: is_active
+ type: ${boolean.type}
+ remarks: True only for currently running DAG runs, null for the others
+ constraints:
+ nullable: true
+ defaultValue: true
+ - column:
+ name: start_time
+ type: ${timestamp_type}
+ remarks: When the DAG run started
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: end_time
+ type: ${timestamp_type}
+ remarks: When the DAG run completed (null if still running)
+ constraints:
+ nullable: true
+ - column:
+ name: message
+ type: ${text.type}
+ remarks: Human-readable message about the run; may contain the error message when the run failed
+ constraints:
+ nullable: true
+ - column:
+ name: user_id
+ type: int
+ remarks: The user who triggered the DAG run (null if the user was later deleted)
+ constraints:
+ nullable: true
+ - column:
+ name: last_heartbeat
+ type: ${timestamp_type}
+ defaultValueComputed: current_timestamp
+ remarks: Heartbeat timestamp stamped periodically while the run executes; a stale value means the owning process died
+ constraints:
+ nullable: false
+ - column:
+ name: created_at
+ type: ${timestamp_type}
+ remarks: The timestamp this DAG run was created.
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: updated_at
+ type: ${timestamp_type}
+ remarks: The timestamp this DAG run was last updated.
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ rollback:
+ - dropTable:
+ tableName: transform_dag_run
+
+ - changeSet:
+ id: v64.btfmlX
+ author: bronsa
+ comment: Index transform_dag_run on source_transform_id for per-transform DAG run history lookups (also backs the FK below)
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_source_transform_id
+ changes:
+ - createIndex:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_source_transform_id
+ columns:
+ - column:
+ name: source_transform_id
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.dQGhvK
+ author: bronsa
+ comment: >-
+ FK from transform_dag_run.source_transform_id to transform; SET NULL on delete so DAG-run
+ history survives the transform (as for transform_run.transform_id), displayable via the
+ run-start name snapshot
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyTableName: transform_dag_run
+ foreignKeyName: fk_transform_dag_run_source_transform_id
+ changes:
+ - addForeignKeyConstraint:
+ constraintName: fk_transform_dag_run_source_transform_id
+ baseTableName: transform_dag_run
+ baseColumnNames: source_transform_id
+ referencedTableName: transform
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.yY6UC9
+ author: bronsa
+ comment: Index transform_dag_run on user_id (backs the FK below and covers per-user run history lookups)
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_user_id
+ changes:
+ - createIndex:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_user_id
+ columns:
+ - column:
+ name: user_id
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.BTINKo
+ author: bronsa
+ comment: FK from transform_dag_run.user_id to core_user; null it out when the user is deleted so run history survives
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyTableName: transform_dag_run
+ foreignKeyName: fk_transform_dag_run_user_id
+ changes:
+ - addForeignKeyConstraint:
+ constraintName: fk_transform_dag_run_user_id
+ baseTableName: transform_dag_run
+ baseColumnNames: user_id
+ referencedTableName: core_user
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.ohN51V
+ author: bronsa
+ comment: Index transform_dag_run on (is_active, last_heartbeat) so the every-minute reaper scan is O(active rows)
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_active_runs
+ changes:
+ - createIndex:
+ tableName: transform_dag_run
+ indexName: idx_transform_dag_run_active_runs
+ columns:
+ - column:
+ name: is_active
+ - column:
+ name: last_heartbeat
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.wQe4Fn
+ author: bronsa
+ comment: >-
+ Unique constraint so at most one DAG run can be active per seed transform.
+ changes:
+ - addUniqueConstraint:
+ tableName: transform_dag_run
+ constraintName: idx_unique_active_transform_dag_run
+ columnNames: source_transform_id, is_active
+ rollback: # nothing to do: rolling back the createTable changeset drops the whole table
+
+ - changeSet:
+ id: v64.HUpfDD
+ author: bronsa
+ comment: >-
+ Add dag_run_id to transform_run to link each transform execution to its coordinating DAG run
+ (mirrors job_run_id, which links to transform_job_run). Exactly one of the two is set for a
+ coordinated member run; both are null for a standalone single-transform run.
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_run
+ columnName: dag_run_id
+ changes:
+ - addColumn:
+ tableName: transform_run
+ columns:
+ - column:
+ name: dag_run_id
+ type: bigint
+ remarks: FK to transform_dag_run; links each transform execution to its coordinating DAG run
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: transform_run
+ columnName: dag_run_id
+
+ - changeSet:
+ id: v64.tOY28l
+ author: bronsa
+ comment: Index transform_run on dag_run_id for efficient drill-down of a DAG run's member transform runs
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - indexExists:
+ tableName: transform_run
+ indexName: idx_transform_run_dag_run_id
+ changes:
+ - createIndex:
+ tableName: transform_run
+ indexName: idx_transform_run_dag_run_id
+ columns:
+ - column:
+ name: dag_run_id
+ rollback:
+ - dropIndex:
+ tableName: transform_run
+ indexName: idx_transform_run_dag_run_id
+
+ - changeSet:
+ id: v64.cH9mZs
+ author: bronsa
+ comment: >-
+ Snapshot the job's name on transform_job_run at run start, as transform_run does for its
+ transform. transform_job_run.job_id has no FK, so job runs already outlive their job — but
+ namelessly; the snapshot keeps them displayable in run history after the job is deleted.
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_job_run
+ columnName: job_name
+ changes:
+ - addColumn:
+ tableName: transform_job_run
+ columns:
+ - column:
+ name: job_name
+ type: ${text.type}
+ remarks: Name of the job at run start, preserved after its deletion
+ rollback:
+ - dropColumn:
+ tableName: transform_job_run
+ columnName: job_name
+
+ - changeSet:
+ id: v64.gW3nQd
+ author: bronsa
+ comment: Snapshot the job's entity_id on transform_job_run at run start, for potential reconnection when a deleted job is recreated
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_job_run
+ columnName: job_entity_id
+ changes:
+ - addColumn:
+ tableName: transform_job_run
+ columns:
+ - column:
+ name: job_entity_id
+ type: char(21)
+ remarks: entity_id of the job at run start, for potential reconnection when it is recreated
+ rollback:
+ - dropColumn:
+ tableName: transform_job_run
+ columnName: job_entity_id
+
+ - changeSet:
+ id: v64.aY5kXm
+ author: bronsa
+ comment: Backfill the job name/entity_id snapshots on existing transform_job_run rows from their live jobs (this table predates the branch)
+ changes:
+ - sql:
+ sql: >-
+ UPDATE transform_job_run
+ SET job_name = (SELECT name FROM transform_job WHERE transform_job.id = transform_job_run.job_id),
+ job_entity_id = (SELECT entity_id FROM transform_job WHERE transform_job.id = transform_job_run.job_id)
+ rollback:
+ - sql:
+ sql: UPDATE transform_job_run SET job_name = NULL, job_entity_id = NULL
diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.clj
index 3c80453c6f5..bf474669e94 100644
--- a/src/metabase/api_routes/routes.clj
+++ b/src/metabase/api_routes/routes.clj
@@ -243,6 +243,7 @@
"/timeline" (+auth metabase.timeline.api/timeline-routes)
"/timeline-event" (+auth metabase.timeline.api/timeline-event-routes)
"/transform" (+auth metabase.transforms-rest.api.transform/routes)
+ "/transform-dag-run" (+auth metabase.transforms-rest.api.transform/transform-dag-run-routes)
"/transform-job" (+auth metabase.transforms-rest.api.transform/transform-job-routes)
"/transform-tag" (+auth metabase.transforms-rest.api.transform/transform-tag-routes)
"/typed-schemas" (+auth metabase.typed-schemas.api/routes)
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 439f80e30d8..21cf56883ae 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -130,6 +130,7 @@
:model/Timeline metabase.timeline.models.timeline
:model/TimelineEvent metabase.timeline.models.timeline-event
:model/Transform metabase.transforms.models.transform
+ :model/TransformDagRun metabase.transforms.models.dag-run
:model/TransformJob metabase.transforms.models.transform-job
:model/TransformJobRun metabase.transforms.models.job-run
:model/TransformJobTransformTag metabase.transforms.models.transform-job-transform-tag
diff --git a/src/metabase/transforms/coordinated_run.clj b/src/metabase/transforms/coordinated_run.clj
new file mode 100644
index 00000000000..258911f6b66
--- /dev/null
+++ b/src/metabase/transforms/coordinated_run.clj
@@ -0,0 +1,99 @@
+(ns metabase.transforms.coordinated-run
+ "Shared lifecycle operations for the two coordinated-run models, `transform_job_run` and
+ `transform_dag_run`. Both track a multi-transform run with the same status/heartbeat lifecycle and
+ differ only in their extra columns and member-run FK; helpers here are parameterized by the Toucan
+ `model`."
+ (:require
+ [metabase.run-tracking.core :as rt]
+ [metabase.transforms.canceling :as canceling]
+ [metabase.transforms.models.transform-run-cancelation :as transform-run-cancelation]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn add-run-activity!
+ "Note that a run has had activity (touches `updated_at`)."
+ [model run-id]
+ (t2/update! model :id run-id :is_active true {:updated_at :%now}))
+
+(defn succeed-started-run!
+ "Mark a started run as successfully completed."
+ ([model run-id]
+ (succeed-started-run! model run-id {}))
+ ([model run-id properties]
+ (t2/update! model
+ :id run-id
+ :is_active true
+ (merge {:end_time :%now}
+ properties
+ {:status :succeeded
+ :is_active nil}))))
+
+(defn fail-started-run!
+ "Mark the started active run as failed and inactive."
+ [model run-id properties]
+ (t2/update! model
+ :id run-id
+ :is_active true
+ (merge {:end_time :%now}
+ properties
+ {:status :failed
+ :is_active nil})))
+
+(defn cancel-started-run!
+ "Mark an active run as canceled; a finished run is never resurrected into a canceled state.
+ Returns the number of rows updated — 0 if the run had already finished."
+ [model run-id]
+ (t2/update! model
+ :id run-id
+ :is_active true
+ {:status :canceled
+ :is_active nil
+ :end_time :%now
+ :message "Canceled"}))
+
+(defn cancel!
+ "Cancel an in-progress coordinated run: mark it canceled and request cancellation of its
+ still-active member transform runs — via the in-process cancel channel, plus a cancelation row so
+ the cancel-runs task reaches members running on other nodes. Returns true if the run was active
+ and is now canceled, false if it had already finished."
+ [model member-fk run-id]
+ (boolean
+ (when (pos? (cancel-started-run! model run-id))
+ (doseq [member-run-id (t2/select-pks-vec :model/TransformRun member-fk run-id :is_active true)]
+ (transform-run-cancelation/mark-cancel-started-run! member-run-id)
+ (canceling/chan-signal-cancel! member-run-id))
+ true)))
+
+(defn heartbeat-runs!
+ "Stamp `last_heartbeat = now` on the given still-active run-ids."
+ [model run-ids]
+ (rt/heartbeat-ids! model [:= :is_active true] :last_heartbeat run-ids))
+
+(defn reap-orphaned-runs!
+ "Time out active runs whose `last_heartbeat` is older than `stale-minutes` (their coordinator
+ process is presumed dead). `type-tag` (\"job\"/\"dag\") tags the emitted metrics. Returns the rows
+ that were timed out so callers can notify."
+ [model type-tag stale-minutes]
+ (rt/reap-orphaned!
+ {:model model
+ :active [:= :is_active true]
+ :stale [:< :last_heartbeat (rt/cutoff stale-minutes :minute)]
+ :terminal {:status "timeout" :end_time :%now :is_active nil :message "Timed out: crashed"}
+ :metrics {:total-metric :metabase-transforms/timeouts-total
+ :latency-metric :metabase-transforms/timeout-detection-latency-ms
+ :tags {:type type-tag}
+ :latency-column :last_heartbeat
+ :timeout-ms (rt/unit->ms stale-minutes :minute)}}))
+
+(defn heartbeat-and-reconcile!
+ "Stamp a heartbeat on every active run whose id is a key of `active-runs-atom` (the runs this
+ process coordinates), then deliver the `gone` promise of any that were terminated externally so
+ their coordinator aborts."
+ [model active-runs-atom]
+ (rt/heartbeat-and-reconcile! {:model model
+ :active [:= :is_active true]
+ :ids (keys @active-runs-atom)
+ :heartbeat! #(heartbeat-runs! model %)
+ :on-gone (fn [run-id]
+ (some-> (get @active-runs-atom run-id) (deliver true)))}))
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index b1db52505e2..cc453bfbffa 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -5,8 +5,10 @@
[metabase.transforms-base.util]
[metabase.transforms.canceling]
[metabase.transforms.crud]
+ [metabase.transforms.dag]
[metabase.transforms.execute]
[metabase.transforms.jobs]
+ [metabase.transforms.models.dag-run]
[metabase.transforms.models.job-run]
[metabase.transforms.models.transform]
[metabase.transforms.models.transform-job]
@@ -14,6 +16,7 @@
[metabase.transforms.models.transform-run-cancelation]
[metabase.transforms.models.transform-tag]
[metabase.transforms.notification]
+ [metabase.transforms.run-listing]
[metabase.transforms.schedule]
[metabase.transforms.settings]
[metabase.transforms.usage]
@@ -57,7 +60,12 @@
get-transform-cycle]
[metabase.transforms.jobs
run-job!
+ cancel-job-run!
job-transforms]
+ [metabase.transforms.dag
+ run-dag!
+ cancel-dag-run!
+ dag-run-transforms]
[metabase.transforms.schedule
validate-cron-expression
initialize-job!
@@ -74,6 +82,11 @@
[metabase.transforms.models.job-run
paged-job-runs
transform-runs-for-job-run]
+ [metabase.transforms.models.dag-run
+ transform-runs-for-dag-run]
+ [metabase.transforms.run-listing
+ paged-run-summaries
+ present-run-summaries]
[metabase.transforms.models.transform-run-cancelation
mark-cancel-started-run!]
[metabase.transforms.models.transform-job
diff --git a/src/metabase/transforms/dag.clj b/src/metabase/transforms/dag.clj
new file mode 100644
index 00000000000..de3d6ae6b5e
--- /dev/null
+++ b/src/metabase/transforms/dag.clj
@@ -0,0 +1,184 @@
+(ns metabase.transforms.dag
+ "Manual DAG-reprocess runs: reprocessing the transitive dependency DAG rooted at a single seed
+ transform, upstream or downstream. This reuses the generic multi-transform coordinator in
+ [[metabase.transforms.jobs]] (which honors the DAG, lanes, cancelation and timeouts) but tracks
+ progress in its own `transform_dag_run` table rather than `transform_job_run`, and links member
+ transform runs via `transform_run.dag_run_id`."
+ (:require
+ [clojure.set :as set]
+ [metabase.run-tracking.core :as rt]
+ [metabase.task.core :as task]
+ [metabase.tracing.core :as tracing]
+ [metabase.transforms-base.ordering :as ordering]
+ [metabase.transforms.coordinated-run :as coordinated-run]
+ [metabase.transforms.jobs :as jobs]
+ [metabase.transforms.models.dag-run :as dag-run]
+ [metabase.transforms.util :as transforms.u]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private dag-heartbeat-stale-minutes
+ "A DAG run whose coordinator hasn't heartbeat in this many minutes is presumed dead and reaped."
+ 5)
+
+(defonce ^:private dag-active-runs
+ ;; dag-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped) so
+ ;; its coordinator aborts. Separate from the job-run atom in [[metabase.transforms.jobs]] because
+ ;; the two id spaces (transform_dag_run vs transform_job_run) are independent.
+ (atom {}))
+
+;;; ------------------------------------------- Planning -------------------------------------------
+
+(defn- dependents-graph
+ "Reverse a forward dependency map (id -> #{deps it reads}) into a dependents map
+ (id -> #{transforms that read it})."
+ [forward-deps]
+ (reduce-kv (fn [acc id deps]
+ (reduce #(update %1 %2 (fnil conj #{}) id) acc deps))
+ {} forward-deps))
+
+(defn- reachable
+ "Set of nodes reachable from `start` in `graph` (id -> #{neighbor-ids}), including `start`."
+ [graph start]
+ (loop [queue [start] seen #{}]
+ (if-let [node (first queue)]
+ (if (seen node)
+ (recur (rest queue) seen)
+ (recur (into (rest queue) (get graph node)) (conj seen node)))
+ seen)))
+
+(defn- lean-transforms
+ "Transforms with only the columns the ordering walk needs — avoids loading every transform's full
+ row just to compute the dependency graph (full rows are fetched only for the resulting closure)."
+ []
+ (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies]))
+
+(defn- full-transforms [ids]
+ (if (seq ids)
+ (t2/select :model/Transform :id [:in ids])
+ []))
+
+(defn- dag-run-plan
+ "Compute both the set of transform-ids a DAG reprocess should run and the execution plan to run
+ them. Returns `{:transform-ids #{...} :plan {:order [...] :deps {...}}}`.
+
+ `:upstream` — seed + all transforms the seed transitively depends on. This is exactly the closure
+ `transform-ordering` walks from the seed, so the walk's `:dependencies` map is both
+ the id set (its keys) and the plan input. The whole closure runs.
+ `:downstream` — seed + all transforms that transitively depend on the seed. The ordering code only
+ walks toward dependencies, so we build the full forward graph once and reverse it to
+ find dependents. Only the downstream set runs — the dependents' *other* upstream
+ inputs are already-materialized tables and must NOT be reprocessed, so we keep only
+ the dependency edges that stay within the downstream set. Dropping the external edges
+ preserves ordering (the seed still runs before anything that depends on it) while
+ leaving those inputs untouched."
+ [seed-id direction]
+ (let [lean (lean-transforms)]
+ (case direction
+ :upstream
+ (let [{:keys [dependencies uncached]} (ordering/transform-ordering #{seed-id} lean)
+ ids (set (keys dependencies))]
+ (ordering/persist-table-dependencies! uncached)
+ {:transform-ids ids
+ :plan (jobs/dependencies->plan dependencies (full-transforms ids))})
+
+ :downstream
+ (let [{:keys [dependencies uncached]} (ordering/transform-ordering (into #{} (map :id) lean) lean)
+ _ (ordering/persist-table-dependencies! uncached)
+ downstream-ids (reachable (dependents-graph dependencies) seed-id)
+ deps (into {}
+ (map (fn [id]
+ [id (set/intersection (get dependencies id #{}) downstream-ids)]))
+ downstream-ids)]
+ {:transform-ids downstream-ids
+ :plan (jobs/dependencies->plan deps (full-transforms downstream-ids))}))))
+
+(defn dag-run-transforms
+ "Return the transforms a DAG reprocess from `transform-id` would run, in planned execution order.
+ `direction` is as in [[run-dag!]]."
+ [transform-id direction]
+ (-> (dag-run-plan transform-id direction) :plan :order vec))
+
+;;; ------------------------------------------- Execution -------------------------------------------
+
+(defn- start-dag-run-unless-racing!
+ "Insert the DAG run row, or return nil if a concurrent trigger for the same seed transform won
+ the `(source_transform_id, is_active)` uniqueness race."
+ [transform-id direction user-id transform-count]
+ (try
+ (dag-run/start-dag-run! transform-id direction user-id transform-count)
+ (catch Exception e
+ (when-not (transforms.u/duplicate-key-violation? e)
+ (throw e)))))
+
+(defn run-dag!
+ "Run all transforms in the dependency DAG rooted at `transform-id`, tracking progress in a
+ `transform_dag_run` row. Blocks until the run finishes; runs are always manual.
+
+ `direction` selects which transforms are included:
+ - `:upstream` — the seed plus all transforms it transitively depends on
+ - `:downstream` — the seed plus all transforms that transitively depend on it
+
+ `:start-promise`, if provided, is delivered `[:started dag-run-id]` once the run row is created,
+ `nil` if nothing was run (already running / empty closure), or a Throwable on a pre-start failure.
+ Returns the dag-run-id, or nil if nothing was executed."
+ [transform-id {:keys [direction user-id start-promise]}]
+ (try
+ (if (dag-run/running-run-for-source-transform-id transform-id)
+ (do (log/info "Not executing DAG run for transform" (pr-str transform-id) "because one is already running")
+ (some-> start-promise (deliver nil))
+ nil)
+ (let [{:keys [transform-ids plan]} (dag-run-plan transform-id direction)]
+ (if (empty? transform-ids)
+ (do (log/info "Skipping DAG run for transform" (pr-str transform-id) "because no transforms found in closure")
+ (some-> start-promise (deliver nil))
+ nil)
+ (if-let [{run-id :id} (start-dag-run-unless-racing! transform-id direction user-id
+ (count transform-ids))]
+ (do (some-> start-promise (deliver [:started run-id]))
+ (tracing/with-span :tasks "task.transform.run-dag" {:transform/id transform-id
+ :transform/direction (name direction)
+ :transform/count (count transform-ids)}
+ (jobs/execute-coordinated-run! :model/TransformDagRun run-id transform-ids plan
+ (format "DAG run for transform %s" (pr-str transform-id))
+ {:run-method :manual
+ :user-id user-id
+ :parent-run-type :dag
+ :active-runs-atom dag-active-runs}))
+ run-id)
+ (do (log/info "Not executing DAG run for transform" (pr-str transform-id) "because one is already running")
+ (some-> start-promise (deliver nil))
+ nil)))))
+ (catch Throwable t
+ ;; unblock any caller waiting on the promise on a pre-start failure
+ (some-> start-promise (deliver t))
+ (throw t))))
+
+(defn cancel-dag-run!
+ "Cancel an in-progress DAG run and request cancellation of its still-running member transform runs.
+ Returns true if the run was active and is now canceled, false if it had already finished."
+ [run-id]
+ (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id))
+
+;;; ------------------------------------------- Heartbeat / reaping -------------------------------------------
+
+(defn- heartbeat-and-reconcile-runs!
+ "Stamp a heartbeat on every active DAG run this process is coordinating, then deliver the `gone`
+ promise of any that were terminated externally so their coordinator aborts."
+ []
+ (coordinated-run/heartbeat-and-reconcile! :model/TransformDagRun dag-active-runs))
+
+(defn- reap-orphaned-runs!
+ "Reap DAG runs whose coordinator process died (stale heartbeat)."
+ []
+ (coordinated-run/reap-orphaned-runs! :model/TransformDagRun "dag" dag-heartbeat-stale-minutes))
+
+(defmethod task/init! ::TransformDagRunHeartbeat [_]
+ (rt/start-heartbeat! heartbeat-and-reconcile-runs! 1))
+
+(defmethod task/init! ::TransformDagRunReaper [_]
+ (rt/schedule-reaper! {:job-key "metabase.transforms.dag.reaper-job"
+ :label "transform DAG run"
+ :reap-fn #'reap-orphaned-runs!}))
diff --git a/src/metabase/transforms/init.clj b/src/metabase/transforms/init.clj
index 3678691726e..484206d4a41 100644
--- a/src/metabase/transforms/init.clj
+++ b/src/metabase/transforms/init.clj
@@ -1,7 +1,9 @@
(ns metabase.transforms.init
(:require
[metabase.transforms.canceling]
+ [metabase.transforms.dag]
[metabase.transforms.jobs]
+ [metabase.transforms.models.dag-run]
[metabase.transforms.models.job-run]
[metabase.transforms.models.transform]
[metabase.transforms.models.transform-job]
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 7125cabfb46..135f13acdb6 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -12,6 +12,7 @@
[metabase.transforms-base.ordering :as transforms-base.ordering]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as canceling]
+ [metabase.transforms.coordinated-run :as coordinated-run]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.freshness :as freshness]
[metabase.transforms.instrumentation :as transforms.instrumentation]
@@ -54,6 +55,32 @@
transform-id-2])))
ordering)))
+(defn dependencies->plan
+ "Build an execution plan from a `dependencies` map (transform-id -> #{dep-ids}, as returned by
+ `transform-ordering`) and the transforms it covers. Returns `{:order [...transforms in execution
+ order] :deps dependencies}`. Throws if the dependency graph contains a cycle.
+
+ This is the pure, in-memory part of planning — no DB or `table-dependencies` work — so callers
+ that already hold a dependency map (e.g. a DAG reprocess that computed the whole graph once) can
+ build a plan without recomputing it."
+ [dependencies all-transforms]
+ (let [transforms-by-id (into {}
+ (keep (fn [{:keys [id] :as transform}]
+ (when (contains? dependencies id)
+ [id transform])))
+ all-transforms)
+ sorted-ord (sorted-ordering dependencies transforms-by-id)]
+ (when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]
+ (let [id->name (into {} (map (juxt :id :name)) all-transforms)]
+ (throw (ex-info (str "Cyclic transform definitions detected: "
+ (str/join " → " (map id->name cycle)))
+ {:cycle cycle}))))
+ (loop [complete (ordered-set/ordered-set)]
+ (if-let [current-transform (next-transform sorted-ord transforms-by-id complete)]
+ (recur (conj complete (:id current-transform)))
+ {:order (map transforms-by-id complete)
+ :deps dependencies}))))
+
(defn- get-plan [transform-ids]
(tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}
(let [all-transforms (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies])
@@ -73,20 +100,10 @@
;; Lazily backfill the table_dependencies column for any closure transform we had to compute live.
(transforms-base.ordering/persist-table-dependencies! uncached)
;; Fetch full rows only for the closure, which is what callers actually consume.
- (let [transforms-by-id (if (seq dependencies)
- (u/index-by :id (t2/select :model/Transform :id [:in (keys dependencies)]))
- {})
- sorted-ord (sorted-ordering dependencies transforms-by-id)]
- (when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]
- (let [id->name (into {} (map (juxt :id :name)) (vals transforms-by-id))]
- (throw (ex-info (str "Cyclic transform definitions detected: "
- (str/join " → " (map id->name cycle)))
- {:cycle cycle}))))
- (loop [complete (ordered-set/ordered-set)]
- (if-let [current-transform (next-transform sorted-ord transforms-by-id complete)]
- (recur (conj complete (:id current-transform)))
- {:order (map transforms-by-id complete)
- :deps dependencies}))))))
+ (let [closure-transforms (if (seq dependencies)
+ (t2/select :model/Transform :id [:in (keys dependencies)])
+ [])]
+ (dependencies->plan dependencies closure-transforms)))))
(defn- wait-for-transform-slot!
"Poll until no active run exists for `transform-id`, up to `timeout-ms`."
@@ -97,7 +114,13 @@
(>= (u/since-ms timer) timeout-ms) false
:else (do (Thread/sleep 2000) (recur)))))
-(defn- run-transform! [run-id run-method user-id started-run-id {transform-id :id :as transform}]
+(defn- run-transform!
+ "Execute a single transform as part of a coordinated run. `ctx` is the coordinator context; this
+ uses its `:parent-run` (a `[type id]` tuple — `[:job id]`/`[:dag id]` — that links the created
+ `transform_run` row to its coordinating `transform_job_run`/`transform_dag_run`), `:run-method`,
+ `:user-id`, and `:add-run-activity!` (a zero-arg fn called after successful execution to touch the
+ coordinating run's updated_at)."
+ [{:keys [parent-run run-method user-id add-run-activity!]} started-run-id {transform-id :id :as transform}]
(cond
(not (transforms.u/check-feature-enabled transform))
(log/warnf "Skip running transform %d due to lacking premium features" transform-id)
@@ -118,10 +141,10 @@
(pr-str transform-id) (transforms.settings/transform-timeout))
{:transform-id transform-id :error :already-running-timeout}))
(let [result (try
- (log/info "Executing job transform" (pr-str transform-id))
+ (log/info "Executing transform" (pr-str transform-id))
(transforms.execute/execute! transform {:run-method run-method
:user-id user-id
- :job-run-id run-id
+ :parent-run parent-run
;; lets the coordinator cancel exactly the run
;; this worker started (see [[cancel-worker!]])
:on-start #(deliver started-run-id %)})
@@ -133,7 +156,7 @@
(throw e))))]
(when (= :already-running result)
(recur))))))
- (transforms.job-run/add-run-activity! run-id))))
+ (add-run-activity!))))
(defn- lane-for
"Lane a transform runs in: `:py` for python transforms (single-slot), `:sql` otherwise."
@@ -149,18 +172,20 @@
"Run `transform` on its own thread; returns a future yielding a `::status` completion map.
`started-run-id` is a promise the worker delivers its transform run id to once `execute!` has
created it — at most once, since the `:already-running` retry only re-enters after a failed
- start — so [[cancel-worker!]] can target exactly that run."
- [run-id run-method user-id started-run-id transform]
+ start — so [[cancel-worker!]] can target exactly that run. `ctx` is the coordinator context,
+ forwarded to [[run-transform!]] "
+ [{[run-type run-id] :parent-run :as ctx} started-run-id transform]
(future
- (try
- (run-transform! run-id run-method user-id started-run-id transform)
- {::status :succeeded ::transform transform}
- (catch Throwable t
- (log/errorf t "Transform %s in run %s failed" (pr-str (:id transform)) (pr-str run-id))
- {::status :failed
- ::transform transform
- ::message (or (ex-message t) (str t))
- ::throwable t}))))
+ (log/with-context {:run-type run-type, :run-id run-id, :transform-id (:id transform)}
+ (try
+ (run-transform! ctx started-run-id transform)
+ {::status :succeeded ::transform transform}
+ (catch Throwable t
+ (log/errorf t "Transform %s in run %s failed" (pr-str (:id transform)) (pr-str run-id))
+ {::status :failed
+ ::transform transform
+ ::message (or (ex-message t) (str t))
+ ::throwable t})))))
(defn- transform-target-key
"A coarse identity for a transform's output table, used to keep two transforms that write the
@@ -173,7 +198,7 @@
(defn- dispatch-ready!
"Submit every transform whose deps have all succeeded and whose lane has a free slot, recording the
transitive cascade of dep-failures along the way. Returns the updated state."
- [st {:keys [plan deps lanes run-id run-method user-id]}]
+ [st {:keys [plan deps lanes] :as ctx}]
(reduce
(fn [{:keys [succeeded failed in-flight-targets] :as st} t]
(let [id (:id t)
@@ -203,7 +228,7 @@
(and (every? succeeded dep-ids)
(< (count in-flight-now) (lanes lane-key)))
(let [started-run-id (promise)
- fut (submit-transform! run-id run-method user-id started-run-id t)]
+ fut (submit-transform! ctx started-run-id t)]
(cond-> (assoc-in st [:in-flight lane-key id]
{:future fut
:timer (u/start-timer)
@@ -270,19 +295,14 @@
(log/warnf t "Error canceling in-flight worker for transform %s" (pr-str id))))))
(defonce ^:private active-runs
- ;; job-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped)
+ ;; job-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped).
(atom {}))
(defn- heartbeat-and-reconcile-runs!
- "Stamp a heartbeat on every job run this process is coordinating, then deliver the `gone` promise
- of any that another path (reaper, force-fail) already terminated, so its coordinator aborts."
+ "Stamp a heartbeat on every active job run this process owns, then deliver the `gone` promise of
+ any that were terminated externally so their coordinator aborts."
[]
- (rt/heartbeat-and-reconcile! {:model :model/TransformJobRun
- :active [:= :is_active true]
- :ids (keys @active-runs)
- :heartbeat! transforms.job-run/heartbeat-runs!
- :on-gone (fn [run-id]
- (some-> (get @active-runs run-id) (deliver true)))}))
+ (coordinated-run/heartbeat-and-reconcile! :model/TransformJobRun active-runs))
(defn- run-coordinator-loop!
"Dispatch ready transforms; then each round sweep the in-flight workers and refill freed slots —
@@ -317,26 +337,34 @@
(:now (t2/query-one {:select [[[:raw "current_timestamp"] :now]]})))
(defn run-transforms!
- "Run `transform-ids-to-run` and their dependencies, honoring the DAG.
+ "Run the transforms of `plan`, honoring the DAG.
Each transform runs on its own thread; SQL transforms run concurrently up to
`transform-run-job-sql-concurrency`, python transforms one at a time (the python-runner service has
a single worker). A failed dependency fails its dependents transitively; a worker that overruns the
transform timeout (plus grace) is interrupted and failed.
- Transforms pulled into the plan only as dependencies (not directly requested) are skipped while
- still fresh, unless `skip-fresh-deps?` is false.
+ `transform-ids-to-run` are the directly requested transforms; the rest of the plan was pulled in
+ as dependencies and is skipped while still fresh, unless `skip-fresh-deps?` is false.
+
+ `:parent-run-type` (`:job`/`:dag`, default `:job`) selects which coordinating run each member
+ `transform_run` links back to.
+ `:active-runs-atom` (default `active-runs`) is the atom that tracks coordinator liveness for heartbeating.
+ `:add-run-activity!` is a zero-arg fn called after each successful transform execution.
Returns `{::status :succeeded}`, `{::status :failed ::failures [...]}`, or `{::status :aborted}`
- when the job run was terminated externally (e.g. reaped) while this coordinator was still running."
- [run-id transform-ids-to-run {:keys [run-method start-promise user-id skip-fresh-deps?]
- :or {skip-fresh-deps? true}}]
+ when the coordinating run was terminated externally (e.g. reaped) while this coordinator was still running."
+ [run-id transform-ids-to-run {plan :order deps :deps} {:keys [run-method user-id skip-fresh-deps?
+ parent-run-type active-runs-atom add-run-activity!]
+ :or {skip-fresh-deps? true
+ parent-run-type :job
+ active-runs-atom active-runs
+ add-run-activity! (constantly nil)}}]
(let [gone (promise)
- _ (swap! active-runs assoc run-id gone)
+ _ (swap! active-runs-atom assoc run-id gone)
final-state
(try
- (let [{plan :order deps :deps} (get-plan transform-ids-to-run)
- requested (set transform-ids-to-run)
+ (let [requested (set transform-ids-to-run)
closure (into #{} (map :id) plan)
;; Only deps pulled into the plan (not directly requested) are freshness-gated. Seeding them
;; as :succeeded lets their dependents dispatch while they themselves are never submitted.
@@ -350,23 +378,23 @@
:in-flight {:sql {} :py {}}
;; target tables currently being written by an in-flight transform
:in-flight-targets #{}}
- ctx {:plan plan
- :deps deps
- :run-id run-id
- :run-gone? #(realized? gone)
- :run-method run-method
- :user-id user-id
+ ctx {:plan plan
+ :deps deps
+ :parent-run [parent-run-type run-id]
+ :run-gone? #(realized? gone)
+ :run-method run-method
+ :user-id user-id
+ :add-run-activity! add-run-activity!
;; lane -> max concurrent workers
- :lanes {:sql (max 1 (transforms.settings/transform-run-job-sql-concurrency))
- :py 1}
- :timeout-ms (+ (u/minutes->ms (transforms.settings/transform-timeout))
- transform-worker-grace-ms)}]
+ :lanes {:sql (max 1 (transforms.settings/transform-run-job-sql-concurrency))
+ :py 1}
+ :timeout-ms (+ (u/minutes->ms (transforms.settings/transform-timeout))
+ transform-worker-grace-ms)}]
(when (seq skip)
(log/infof "Skipping %d fresh pulled-in dependency transform(s): %s" (count skip) (pr-str skip)))
- (when start-promise (deliver start-promise :started))
(run-coordinator-loop! init-state ctx))
(finally
- (swap! active-runs dissoc run-id)))]
+ (swap! active-runs-atom dissoc run-id)))]
(cond
(:aborted? final-state) {::status :aborted}
(seq (:failures final-state)) {::status :failed ::failures (:failures final-state)}
@@ -395,7 +423,10 @@
:scheduled (contains? scheduled id))))
plan)))
-(defn- compile-transform-failure-messages [failures]
+(defn- compile-transform-failure-messages
+ "Render the `::failures` of a coordinated run (as returned by [[run-transforms!]]) into a single
+ human-readable message, one block per failed transform with a link to its run."
+ [failures]
(->> failures
(map (fn [failure]
(format "%s %s:\n%s"
@@ -404,6 +435,34 @@
(::message failure))))
(str/join "\n\n")))
+(defn execute-coordinated-run!
+ "Run `transform-ids` per `plan` (as returned by [[get-plan]] or [[dependencies->plan]]) as the
+ members of the coordinated run `run-id` (a row of `model`) and move the row to its terminal
+ state, failing it if execution throws (the throwable is rethrown). `label` names the run in
+ logs; `run-opts` are passed through to [[run-transforms!]]. Returns the [[run-transforms!]]
+ result so callers can act on failures."
+ [model run-id transform-ids plan label run-opts]
+ (try
+ (let [result (run-transforms! run-id transform-ids plan
+ (assoc run-opts
+ :add-run-activity! #(coordinated-run/add-run-activity! model run-id)))]
+ (case (::status result)
+ :succeeded (coordinated-run/succeed-started-run! model run-id)
+ ;; terminated externally (e.g. reaped): the row is already terminal, just log
+ :aborted (log/warnf "%s %s was terminated externally; coordinator aborted." label (pr-str run-id))
+ :failed (try
+ (coordinated-run/fail-started-run! model run-id
+ {:message (compile-transform-failure-messages (::failures result))})
+ (catch Exception e
+ (log/errorf e "Error when failing %s %s." label (pr-str run-id)))))
+ result)
+ (catch Throwable t
+ (try
+ (coordinated-run/fail-started-run! model run-id {:message (ex-message t)})
+ (catch Exception e
+ (log/errorf e "Error when failing %s." label)))
+ (throw t))))
+
(defn- active-users-to-edit-transform
"Return the users that edited the transform, in reverse chron.
Only returns active users.
@@ -467,43 +526,61 @@
failures)}))))
(defn run-job!
- "Runs all transforms for a given job and their dependencies."
- [job-id {:keys [run-method] :as opts}]
- (if (transforms.job-run/running-run-for-job-id job-id)
- (log/info "Not executing transform job" (pr-str job-id) "because it is already running")
- (let [transforms (job-transform-ids job-id)]
- (if (empty? transforms)
- (log/info "Skipping transform job" (pr-str job-id) "because it has no transforms to run")
- (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
- (tracing/with-span :tasks "task.transform.run-job" {:transform.job/id job-id
- :transform.job/run-method (name run-method)
- :transform.job/count (count transforms)}
- (transforms.instrumentation/with-job-timing [job-id run-method]
- (try ;; catch any catastrophic problems
- (let [result (run-transforms! run-id transforms opts)]
- (case (::status result)
- :succeeded (transforms.job-run/succeed-started-run! run-id)
- ;; terminated externally (e.g. reaped): the row is already terminal and the
- ;; terminator already notified, so just log
- :aborted (log/warnf "Transform job run %s for job %s was terminated externally; coordinator aborted."
- (pr-str run-id) (pr-str job-id))
- :failed (try
- (transforms.job-run/fail-started-run! run-id {:message (compile-transform-failure-messages (::failures result))})
- (when (= :cron run-method)
- (notify-transform-failures job-id (::failures result)))
- (catch Exception e
- (log/error e "Error when failing a transform run.")))))
- (catch Throwable t
- ;; We don't expect a catastrophic failure, but neither did the Titanic.
- (try
- (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
- (when (and (::transform-failure (ex-data t))
- (= :cron run-method)) ;; Catastrophic job failures are included in the digest
- (notify-transform-failures job-id (::failures (ex-data t))))
- (catch Exception e
- (log/error e "Error when failing a transform job run.")))
- (throw t)))))
- run-id)))))
+ "Runs all transforms for a given job and their dependencies.
+
+ `:start-promise`, if provided, is delivered `[:started run-id]` once the run row is created, `nil`
+ if nothing was run (already running / no transforms), or a Throwable on a pre-start failure.
+ Returns the run id, or nil if nothing was executed."
+ [job-id {:keys [run-method start-promise] :as opts}]
+ (try
+ (if (transforms.job-run/running-run-for-job-id job-id)
+ (do (log/info "Not executing transform job" (pr-str job-id) "because it is already running")
+ (some-> start-promise (deliver nil))
+ nil)
+ (let [transforms (job-transform-ids job-id)]
+ (if (empty? transforms)
+ (do (log/info "Skipping transform job" (pr-str job-id) "because it has no transforms to run")
+ (some-> start-promise (deliver nil))
+ nil)
+ (let [plan (get-plan transforms)
+ {run-id :id} (transforms.job-run/start-run! job-id run-method)
+ ;; failed cron runs are included in the failure digest
+ notify! (fn [failures]
+ (when (= :cron run-method)
+ (try
+ (notify-transform-failures job-id failures)
+ (catch Exception e
+ (log/errorf e "Error notifying failures of transform job %s" (pr-str job-id))))))]
+ (some-> start-promise (deliver [:started run-id]))
+ (tracing/with-span :tasks "task.transform.run-job" {:transform.job/id job-id
+ :transform.job/run-method (name run-method)
+ :transform.job/count (count transforms)}
+ (transforms.instrumentation/with-job-timing [job-id run-method]
+ (try
+ (let [result (execute-coordinated-run! :model/TransformJobRun run-id transforms plan
+ (format "Transform job run for job %s" (pr-str job-id))
+ (-> opts
+ (dissoc :start-promise)
+ (assoc :parent-run-type :job :active-runs-atom active-runs)))]
+ (when (= (::status result) :failed)
+ (notify! (::failures result))))
+ (catch Throwable t
+ (when (::transform-failure (ex-data t))
+ (notify! (::failures (ex-data t))))
+ (throw t)))))
+ run-id))))
+ (catch Throwable t
+ ;; unblock any caller waiting on the promise on a pre-start failure
+ (some-> start-promise (deliver t))
+ (throw t))))
+
+(defn cancel-job-run!
+ "Cancel an in-progress job run and request cancellation of its still-running member transform runs.
+ Returns true if the run was active and is now canceled, false if it had already finished."
+ [run-id]
+ (coordinated-run/cancel! :model/TransformJobRun :job_run_id run-id))
+
+;;; ------------------------------------------- Orphan reaping -------------------------------------------
(defn- reap-orphaned-runs!
"Reap job runs whose coordinator process died (stale heartbeat)."
diff --git a/src/metabase/transforms/models/dag_run.clj b/src/metabase/transforms/models/dag_run.clj
new file mode 100644
index 00000000000..6d3edb6c89a
--- /dev/null
+++ b/src/metabase/transforms/models/dag_run.clj
@@ -0,0 +1,53 @@
+(ns metabase.transforms.models.dag-run
+ "A `transform_dag_run` tracks a manual DAG-reprocess run: reprocessing the transitive dependency DAG
+ rooted at a single seed transform (upstream or downstream). Like a scheduled job run it coordinates
+ many member transform runs, but it is always triggered manually from one transform rather than by a
+ job — so it has its own table rather than overloading `transform_job_run`. Member runs link back
+ via `transform_run.dag_run_id`."
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(methodical/defmethod t2/table-name :model/TransformDagRun [_model] :transform_dag_run)
+
+(derive :model/TransformDagRun :metabase/model)
+(derive :model/TransformDagRun :hook/timestamped?)
+
+(t2/deftransforms :model/TransformDagRun
+ {:status mi/transform-keyword
+ :direction mi/transform-keyword})
+
+(defn start-dag-run!
+ "Start a DAG-reprocess run seeded from `source-transform-id`, traversing `direction`
+ (`:upstream`/`:downstream`). `user-id` is the user who triggered it; `transform-count` is the
+ number of transforms the run's plan selected to run, stored for display in run listings.
+ Snapshots the seed transform's name and entity_id so the run stays displayable after the
+ transform is deleted (its FK is SET NULL, as for `transform_run.transform_id`)."
+ [source-transform-id direction user-id transform-count]
+ (let [transform (t2/select-one [:model/Transform :name :entity_id] :id source-transform-id)]
+ (t2/insert-returning-instance! :model/TransformDagRun
+ {:source_transform_id source-transform-id
+ :source_transform_name (:name transform)
+ :source_transform_entity_id (:entity_id transform)
+ :direction direction
+ :transform_count transform-count
+ :user_id user-id
+ :status :started
+ :is_active true})))
+
+(defn running-run-for-source-transform-id
+ "Return the single active DAG run seeded from `source-transform-id`, or nil."
+ [source-transform-id]
+ (t2/select-one :model/TransformDagRun
+ :source_transform_id source-transform-id
+ :is_active true))
+
+(defn transform-runs-for-dag-run
+ "Return the transform runs that were part of the given DAG run, ordered by start time."
+ [dag-run-id]
+ (t2/select :model/TransformRun
+ {:where [:= :dag_run_id dag-run-id]
+ :order-by [[:start_time :asc]]}))
diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.clj
index ef107d4dd3e..263afdfbb6b 100644
--- a/src/metabase/transforms/models/job_run.clj
+++ b/src/metabase/transforms/models/job_run.clj
@@ -1,7 +1,7 @@
(ns metabase.transforms.models.job-run
(:require
[metabase.models.interface :as mi]
- [metabase.run-tracking.core :as rt]
+ [metabase.transforms.coordinated-run :as coordinated-run]
[metabase.transforms.models.util :as transforms.models.u]
[methodical.core :as methodical]
[toucan2.core :as t2]
@@ -36,65 +36,25 @@
(t2/reducible-select :model/TransformJobRun (latest-runs-query job-ids)))))
(defn start-run!
- "Start a run"
+ "Start a run. Snapshots the job's name and entity_id so the run stays displayable after the job
+ is deleted (`transform_job_run.job_id` has no FK; job runs outlive their job)."
([job-id run-method]
- (t2/insert-returning-instance! :model/TransformJobRun
- {:job_id job-id
- :run_method run-method
- :status :started
- :is_active true})))
-
-(defn add-run-activity!
- "Notes that a run has had activity"
- [run-id]
- (t2/update! :model/TransformJobRun
- :id run-id
- :is_active true
- {:updated_at :%now}))
-
-(defn succeed-started-run!
- "Mark a started run as successfully completed."
- ([run-id]
- (succeed-started-run! run-id {}))
- ([run-id properties]
- (t2/update! :model/TransformJobRun
- :id run-id
- :is_active true
- (merge {:end_time :%now}
- properties
- {:status :succeeded
- :is_active nil}))))
-
-(defn fail-started-run!
- "Mark the started active run as failed and inactive."
- [run-id properties]
- (t2/update! :model/TransformJobRun
- :id run-id
- :is_active true
- (merge {:end_time :%now}
- properties
- {:status :failed
- :is_active nil})))
-
-(defn heartbeat-runs!
- "Stamp `last_heartbeat = now` on the given still-active job-run-ids."
- [run-ids]
- (rt/heartbeat-ids! :model/TransformJobRun [:= :is_active true] :last_heartbeat run-ids))
+ ;; :built_in_type so the after-select hook localizes built-in job names; str realizes the
+ ;; LocalizedString into the snapshot
+ (let [job (t2/select-one [:model/TransformJob :name :entity_id :built_in_type] :id job-id)]
+ (t2/insert-returning-instance! :model/TransformJobRun
+ {:job_id job-id
+ :job_name (some-> (:name job) str)
+ :job_entity_id (:entity_id job)
+ :run_method run-method
+ :status :started
+ :is_active true}))))
(defn reap-orphaned-runs!
"Time out active job runs whose `last_heartbeat` is older than `stale-minutes` (their coordinator
process is presumed dead). Returns the rows that were timed out so callers can notify."
[stale-minutes]
- (rt/reap-orphaned!
- {:model :model/TransformJobRun
- :active [:= :is_active true]
- :stale [:< :last_heartbeat (rt/cutoff stale-minutes :minute)]
- :terminal {:status "timeout" :end_time :%now :is_active nil :message "Timed out: crashed"}
- :metrics {:total-metric :metabase-transforms/timeouts-total
- :latency-metric :metabase-transforms/timeout-detection-latency-ms
- :tags {:type "job"}
- :latency-column :last_heartbeat
- :timeout-ms (rt/unit->ms stale-minutes :minute)}}))
+ (coordinated-run/reap-orphaned-runs! :model/TransformJobRun "job" stale-minutes))
(defn running-run-for-job-id
"Return a single active job run or nil."
@@ -107,32 +67,24 @@
"Return a page of the list of job runs.
Follows the conventions used by the FE."
- [{:keys [offset limit sort-column sort-direction job-id status run-method start-time]}]
- (let [offset (or offset 0)
- limit (or limit 20)
- sort-direction (or (keyword sort-direction) :desc)
- nulls-sort (if (= sort-direction :asc) :nulls-last :nulls-first)
- sort-column (keyword sort-column)
- order-by (case sort-column
- :start_time [[sort-column sort-direction]]
- :end_time [[sort-column sort-direction nulls-sort]]
- [[:start_time sort-direction]
- [:end_time sort-direction nulls-sort]])
- where-cond (cond-> []
- job-id (conj [:= :job_id job-id])
- status (conj [:= :status status])
- (= status "started") (conj [:= :is_active true])
- run-method (conj [:= :run_method run-method])
- start-time (conj (transforms.models.u/timestamp-constraint :start_time start-time)))
- where (when (seq where-cond) (into [:and] where-cond))
- query-opts (cond-> {:order-by order-by :offset offset :limit limit}
- where (assoc :where where))
- count-opts (if where {:where where} {})
- runs (t2/select :model/TransformJobRun query-opts)]
- {:data runs
+ [{:keys [sort-column sort-direction job-id status run-method start-time offset limit]}]
+ (let [offset (or offset 0)
+ limit (or limit 20)
+ where-cond (cond-> []
+ job-id (conj [:= :job_id job-id])
+ status (conj [:= :status status])
+ (= status "started") (conj [:= :is_active true])
+ run-method (conj [:= :run_method run-method])
+ start-time (conj (transforms.models.u/timestamp-constraint :start_time start-time)))
+ where (when (seq where-cond) (into [:and] where-cond))]
+ {:data (t2/select :model/TransformJobRun
+ (cond-> {:order-by (transforms.models.u/run-order-by sort-column sort-direction)
+ :offset offset
+ :limit limit}
+ where (assoc :where where)))
:limit limit
:offset offset
- :total (t2/count :model/TransformJobRun count-opts)}))
+ :total (t2/count :model/TransformJobRun (if where {:where where} {}))}))
(defn transform-runs-for-job-run
"Return transform runs that were part of the given job run, ordered by start time."
diff --git a/src/metabase/transforms/models/util.clj b/src/metabase/transforms/models/util.clj
index 04846a9eefc..9dfdc573c33 100644
--- a/src/metabase/transforms/models/util.clj
+++ b/src/metabase/transforms/models/util.clj
@@ -22,3 +22,17 @@
(into [:and] (remove nil?)
[(when start [:>= field-name start])
(when end [:< field-name end])])))
+
+(defn run-order-by
+ "Standard `:order-by` clause for a paged run listing. Sorts by `sort-column` (`:start_time` or
+ `:end_time`; anything else — including nil — falls back to ordering by start_time then end_time)
+ in `sort-direction` (`:asc`/`:desc`, defaulting to `:desc`), with in-progress rows (null
+ `end_time`) always ordered last."
+ [sort-column sort-direction]
+ (let [sort-direction (or (keyword sort-direction) :desc)
+ nulls-sort (if (= sort-direction :asc) :nulls-last :nulls-first)]
+ (case (keyword sort-column)
+ :start_time [[:start_time sort-direction]]
+ :end_time [[:end_time sort-direction nulls-sort]]
+ [[:start_time sort-direction]
+ [:end_time sort-direction nulls-sort]])))
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index 65fde1978a7..9efc6fbf2c6 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -32,7 +32,7 @@
(defn- run-mbql-transform!
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target owner_user_id creator_id] :as transform}
- {:keys [run-method on-start user-id job-run-id]}]
+ {:keys [run-method on-start user-id parent-run]}]
;; `:target` is already workspace-rewritten — `resolve-transform-target` runs in
;; `metabase.transforms.execute/execute!` before dispatch.
(try
@@ -42,7 +42,8 @@
run-user-id (if (and (= run-method :manual) user-id)
user-id
(or owner_user_id creator_id))
- {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id :job-run-id job-run-id)]
+ {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id
+ :parent-run parent-run)]
(when on-start (on-start run-id))
(driver.conn/with-write-connection
(log/info "Executing transform" id "with target" (pr-str target)
diff --git a/src/metabase/transforms/run_listing.clj b/src/metabase/transforms/run_listing.clj
new file mode 100644
index 00000000000..756c4ce34c6
--- /dev/null
+++ b/src/metabase/transforms/run_listing.clj
@@ -0,0 +1,153 @@
+(ns metabase.transforms.run-listing
+ "A single paged listing over the three kinds of root run:
+
+ - a job run (`transform_job_run`)
+ - a manual DAG-reprocess run (`transform_dag_run`)
+ - a standalone transform run (`transform_run` not belonging to a job or DAG run)
+
+ Member transform runs that belong to a job or DAG run are excluded — they are reachable through
+ their parent run. Because ids are per-table, a row is identified by the `(run_type, id)` pair;
+ `entity_id` is the id of the associated job/transform."
+ (:require
+ [medley.core :as m]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.models.util :as transforms.models.u]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def run-types
+ "The `run_type` discriminator values a listing row can have."
+ [:job :dag :transform])
+
+;;; ---------------------------------------------- Per-source subqueries ----------------------------------------------
+
+;; Each branch must project the same columns in the same order for the UNION ALL to line up;
+;; `[nil :col]` fills in columns a table lacks.
+
+(defn- job-run-subquery [transform-ids]
+ {:select [[[:inline "job"] :run_type]
+ :id
+ [:job_id :entity_id]
+ [:job_name :entity_name]
+ [nil :direction]
+ [nil :transform_count]
+ :run_method
+ :status :is_active :start_time :end_time :message
+ [nil :user_id]]
+ :from [:transform_job_run]
+ :where (if (seq transform-ids)
+ ;; only job runs that actually ran one of these transforms
+ [:exists {:select [[[:inline 1]]]
+ :from [[:transform_run :member]]
+ :where [:and
+ [:= :member.job_run_id :transform_job_run.id]
+ [:in :member.transform_id transform-ids]]}]
+ true)})
+
+(defn- dag-run-subquery [transform-ids]
+ {:select [[[:inline "dag"] :run_type]
+ :id
+ [:source_transform_id :entity_id]
+ [:source_transform_name :entity_name]
+ :direction
+ :transform_count
+ [[:inline "manual"] :run_method]
+ :status :is_active :start_time :end_time :message
+ :user_id]
+ :from [:transform_dag_run]
+ :where (if (seq transform-ids)
+ [:exists {:select [[[:inline 1]]]
+ :from [[:transform_run :member]]
+ :where [:and
+ [:= :member.dag_run_id :transform_dag_run.id]
+ [:in :member.transform_id transform-ids]]}]
+ true)})
+
+(defn- transform-run-subquery [transform-ids]
+ {:select [[[:inline "transform"] :run_type]
+ :id
+ [:transform_id :entity_id]
+ [:transform_name :entity_name]
+ [nil :direction]
+ [nil :transform_count]
+ :run_method
+ :status :is_active :start_time :end_time :message
+ :user_id]
+ :from [:transform_run]
+ ;; standalone runs only: those not coordinated by a job or DAG run
+ :where (cond-> [:and
+ [:= :job_run_id nil]
+ [:= :dag_run_id nil]]
+ (seq transform-ids) (conj [:in :transform_id transform-ids]))})
+
+(defn- union-subquery
+ "The UNION ALL of the branches selected by `types` (a subset of [[run-types]]), each optionally
+ narrowed to runs touching one of `transform-ids`."
+ [types transform-ids]
+ (let [types (set (or (seq types) run-types))]
+ {:union-all (cond-> []
+ (:job types) (conj (job-run-subquery transform-ids))
+ (:dag types) (conj (dag-run-subquery transform-ids))
+ (:transform types) (conj (transform-run-subquery transform-ids)))}))
+
+(defn paged-run-summaries
+ "Return a page of root runs as a `{:data :limit :offset :total}` envelope. Rows are raw — see
+ [[present-run-summaries]].
+
+ Options (filter semantics match [[metabase.transforms.models.transform-run/paged-runs]]):
+ - `:types` subset of [[run-types]] to include; nil/empty means all three
+ - `:statuses` match any of these status strings
+ - `:run-methods` match any of these triggers (`\"manual\"`/`\"cron\"`)
+ - `:start-time` date range string (as in the QP date parameters) constraining `start_time`
+ - `:end-time` likewise for `end_time`
+ - `:transform-ids` only runs that ran any of these transforms (job/DAG runs whose members include
+ one, or the standalone runs of them)
+ - `:sort-column` `\"start_time\"` / `\"end_time\"`
+ - `:sort-direction` `\"asc\"` / `\"desc\"`
+ - `:offset`/`:limit` pagination (default 0 / 20)"
+ [{:keys [types statuses run-methods start-time end-time transform-ids sort-column sort-direction offset limit]}]
+ (let [offset (or offset 0)
+ limit (or limit 20)
+ inner (union-subquery types transform-ids)
+ where (into [:and] (remove nil?)
+ [(when (seq statuses) [:in :status (set statuses)])
+ ;; started ⇒ still active, as in the per-table run listings
+ (when (= (set statuses) #{"started"}) [:= :is_active true])
+ (when (seq run-methods) [:in :run_method (set run-methods)])
+ (when start-time (transforms.models.u/timestamp-constraint :start_time start-time))
+ (when end-time (transforms.models.u/timestamp-constraint :end_time end-time))])
+ where (when (> (count where) 1) where)
+ base (cond-> {:from [[inner :runs]]}
+ where (assoc :where where))]
+ {:data (t2/query (merge base
+ {:select [:*]
+ :order-by (transforms.models.u/run-order-by sort-column sort-direction)
+ :limit limit
+ :offset offset}))
+ :limit limit
+ :offset offset
+ :total (:count (first (t2/query (merge base {:select [[[:count :*] :count]]}))))}))
+
+(defn present-run-summaries
+ "Prepare raw summary rows for an API response: hydrate each row's `:name` (nil when the underlying
+ job/transform was deleted), keywordize the discriminator columns, and localize timestamps."
+ [rows]
+ (let [by-type (group-by :run_type rows)
+ job-ids (seq (keep :entity_id (get by-type "job")))
+ transform-ids (seq (keep :entity_id (concat (get by-type "dag") (get by-type "transform"))))
+ job->name (when job-ids (t2/select-pk->fn :name :model/TransformJob :id [:in job-ids]))
+ transform->name (when transform-ids (t2/select-pk->fn :name :model/Transform :id [:in transform-ids]))]
+ (map (fn [{:keys [run_type entity_id entity_name] :as row}]
+ (-> row
+ (assoc :name (or (if (= run_type "job")
+ (get job->name entity_id)
+ (get transform->name entity_id))
+ entity_name))
+ (dissoc :entity_name)
+ (update :run_type keyword)
+ (update :status keyword)
+ (m/update-existing :run_method #(some-> % keyword))
+ (m/update-existing :direction #(some-> % keyword))
+ transforms-base.u/localize-run-timestamps))
+ rows)))
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index 08962eb05ba..442c42db2da 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -104,7 +104,7 @@
;;; ------------------------------------------------- Scheduled Execution -------------------------------------------------
-(defn- duplicate-key-violation?
+(defn duplicate-key-violation?
"Check if an exception is a duplicate key violation.
Returns true for Postgres, MySQL/MariaDB, and H2 duplicate key errors."
[e]
@@ -117,19 +117,22 @@
"Start a transform run. Throws ex-info with {:error :already-running} if another
run is already active (duplicate key violation). Other errors are rethrown.
If `user-id` is provided, it will be stored with the run for attribution purposes.
- If `job-run-id` is provided, it will be stored with the run to link it to its parent job run."
- [id run-method user-id & {:keys [job-run-id]}]
- (try
- (transform-run/start-run! id (cond-> {:run_method run-method}
- user-id (assoc :user_id user-id)
- job-run-id (assoc :job_run_id job-run-id)))
- (catch Exception e
- (if (duplicate-key-violation? e)
- (throw (ex-info "Transform is already running"
- {:error :already-running
- :transform-id id}
- e))
- (throw e)))))
+ `parent-run`, when provided, links the run to its coordinating run as a `[type id]` tuple:
+ `[:job id]` is stored in `job_run_id`, `[:dag id]` in `dag_run_id`."
+ [id run-method user-id & {:keys [parent-run]}]
+ (let [[parent-type parent-id] parent-run]
+ (try
+ (transform-run/start-run! id (cond-> {:run_method run-method}
+ user-id (assoc :user_id user-id)
+ (= parent-type :job) (assoc :job_run_id parent-id)
+ (= parent-type :dag) (assoc :dag_run_id parent-id)))
+ (catch Exception e
+ (if (duplicate-key-violation? e)
+ (throw (ex-info "Transform is already running"
+ {:error :already-running
+ :transform-id id}
+ e))
+ (throw e))))))
(defn run-cancelable-transform!
"Execute a transform with cancellation support and proper error handling.
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 6eea839638e..3e71ab33473 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -6,16 +6,16 @@
[metabase.api.util.handlers :as handlers]
[metabase.request.core :as request]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms-rest.api.transform-dag-run :as transforms.dag-run]
[metabase.transforms-rest.api.transform-job]
[metabase.transforms-rest.api.transform-tag]
+ [metabase.transforms-rest.api.util :as transforms-rest.api.u]
[metabase.transforms.core :as transforms.core]
[metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
- [metabase.util.i18n :refer [deferred-tru]]
- [metabase.util.jvm :as u.jvm]
+ [metabase.util.i18n :refer [deferred-tru LocalizedString]]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
- [ring.util.response :as response]
[toucan2.core :as t2])
(:import
(java.time OffsetDateTime)))
@@ -68,6 +68,7 @@
[:transform_name {:optional true} [:maybe :string]]
[:transform_entity_id {:optional true} [:maybe :string]]
[:job_run_id {:optional true} [:maybe pos-int?]]
+ [:dag_run_id {:optional true} [:maybe pos-int?]]
[:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
[:checkpoint_lo_value {:optional true} [:maybe :string]]
[:checkpoint_hi_value {:optional true} [:maybe :string]]
@@ -122,6 +123,7 @@
[:transform_name {:optional true} [:maybe :string]]
[:transform_entity_id {:optional true} [:maybe :string]]
[:job_run_id {:optional true} [:maybe pos-int?]]
+ [:dag_run_id {:optional true} [:maybe pos-int?]]
[:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
[:checkpoint_lo_value {:optional true} [:maybe :string]]
[:checkpoint_hi_value {:optional true} [:maybe :string]]
@@ -215,7 +217,7 @@
[:map
[:sort-column {:optional true} [:enum "transform-name" "start-time" "end-time" "status" "run-method" "transform-tags" "duration"]]
[:sort-direction {:optional true} [:enum "asc" "desc"]]
- [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
+ [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]
[:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
[:transform-tag-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
[:start-time {:optional true} [:maybe ms/NonBlankString]]
@@ -228,6 +230,56 @@
:limit (request/limit)))
(update :data #(map transforms-base.u/present-run %))))
+(def ^:private RunSummaryResponse
+ "One row of the unified runs listing: a job run, a manual DAG-reprocess run, or a standalone
+ transform run (one not belonging to a job/DAG run). Identified by `(run_type, id)`; `entity_id`
+ is the id of the associated job/transform (nil if it was deleted) and `name` its name — live if
+ it still exists, otherwise the name snapshotted at run start. `direction` and `transform_count`
+ (the number of transforms the run's plan selected to run) are set only for DAG runs."
+ [:map {:closed true}
+ [:run_type [:enum :job :dag :transform]]
+ [:id pos-int?]
+ [:entity_id [:maybe pos-int?]]
+ [:name [:maybe [:or :string LocalizedString]]]
+ [:direction [:maybe [:enum :upstream :downstream]]]
+ [:transform_count [:maybe pos-int?]]
+ [:run_method [:maybe :keyword]]
+ [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]
+ [:is_active [:maybe :boolean]]
+ [:start_time :any]
+ [:end_time {:optional true} [:maybe :any]]
+ [:message [:maybe :string]]
+ [:user_id [:maybe pos-int?]]])
+
+(api.macros/defendpoint :get "/runs" :- [:map {:closed true}
+ [:data [:sequential RunSummaryResponse]]
+ [:limit pos-int?]
+ [:offset :int]
+ [:total :int]]
+ "Paginated unified run history: every row is a root run — a job run, a manual DAG-reprocess run,
+ or a standalone transform run — never a member run of a job/DAG (those are listed by `GET /run`
+ and the per-run `transform-runs` endpoints).
+
+ `types` selects which kinds to include (all by default); `transform-ids` narrows to runs that ran
+ any of the given transforms. The remaining filters work as in `GET /run`."
+ [_route-params
+ query-params :-
+ [:map
+ [:types {:optional true} [:maybe (ms/QueryVectorOf [:enum "job" "dag" "transform"])]]
+ [:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout" "canceled" "canceling"])]]
+ [:run-methods {:optional true} [:maybe (ms/QueryVectorOf [:enum "manual" "cron"])]]
+ [:start-time {:optional true} [:maybe ms/NonBlankString]]
+ [:end-time {:optional true} [:maybe ms/NonBlankString]]
+ [:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]
+ [:sort-column {:optional true} [:maybe [:enum "start_time" "end_time"]]]
+ [:sort-direction {:optional true} [:maybe [:enum "asc" "desc"]]]]]
+ (api/check-data-analyst)
+ (-> (transforms.core/paged-run-summaries (assoc query-params
+ :types (map keyword (:types query-params))
+ :offset (request/offset)
+ :limit (request/limit)))
+ (update :data transforms.core/present-run-summaries)))
+
(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse
"Get a transform run by ID."
[{:keys [run-id]} :- [:map
@@ -290,27 +342,27 @@
(t2/update! :model/Transform id {:last_checkpoint_value nil})
nil)
-(defn run-transform!
- "Run a transform. Returns a 202 response with run_id.
- The transform must already be fetched and validated."
+(defn- check-feature-and-lock!
+ "Check that the transform's premium features are enabled and that transforms are not locked by the
+ trial quota."
[transform]
(transforms.core/check-feature-enabled! transform)
(api/check (not (transforms.core/transform-locked? transform))
[402 {:message (deferred-tru "Transforms are temporarily locked because the trial quota has been reached.")
- :error-code "metabase_transforms_locked"}])
- (let [start-promise (promise)]
- (u.jvm/in-virtual-thread*
+ :error-code "metabase_transforms_locked"}]))
+
+(defn run-transform!
+ "Run a transform. Returns a 202 response with run_id.
+ The transform must already be fetched and validated."
+ [transform]
+ (check-feature-and-lock! transform)
+ (transforms-rest.api.u/async-run-response
+ (deferred-tru "Transform run started")
+ :run_id
+ (fn [start-promise]
(transforms.core/execute! transform {:start-promise start-promise
- :run-method :manual
- :user-id api/*current-user-id*}))
- (when (instance? Throwable @start-promise)
- (throw @start-promise))
- (let [result @start-promise
- run-id (when (and (vector? result) (= (first result) :started))
- (second result))]
- (-> (response/response {:message (deferred-tru "Transform run started")
- :run_id run-id})
- (assoc :status 202)))))
+ :run-method :manual
+ :user-id api/*current-user-id*}))))
(api.macros/defendpoint :post "/:id/run" :- [:map
[:status [:= 202]]
@@ -322,11 +374,53 @@
[:id ms/PositiveInt]]]
(run-transform! (api/read-check :model/Transform id)))
+(api.macros/defendpoint :post "/:id/run-dag" :- [:map
+ [:status [:= 202]]
+ [:body [:map {:closed true}
+ [:message :any]
+ [:dag_run_id [:maybe pos-int?]]]]]
+ "Trigger a DAG-reprocess run starting from a single transform: runs the transform and every
+ transform in its transitive dependency closure. Returns a 202 with the created `dag_run_id`, or a
+ nil `dag_run_id` when nothing was run (a DAG run for this transform is already in progress, or the
+ closure is empty).
+
+ `direction` selects which transforms are included:
+ - `upstream` — the seed transform plus all transforms it depends on
+ - `downstream` — the seed transform plus all transforms that depend on it"
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ _query-params
+ {:keys [direction]} :- [:map
+ [:direction (ms/enum-decode-keyword transforms.dag-run/dag-directions)]]]
+ (check-feature-and-lock! (api/write-check :model/Transform id))
+ (transforms-rest.api.u/async-run-response
+ (deferred-tru "DAG run started")
+ :dag_run_id
+ (fn [start-promise]
+ (transforms.core/run-dag! id {:direction direction
+ :user-id api/*current-user-id*
+ :start-promise start-promise}))))
+
+(api.macros/defendpoint :get "/:id/dag-transforms" :- [:sequential [:map {:closed true}
+ [:id pos-int?]
+ [:name :string]]]
+ "Preview the transforms a DAG reprocess from this transform would run (see `POST /:id/run-dag`),
+ in execution order."
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ {:keys [direction]} :- [:map [:direction (ms/enum-decode-keyword transforms.dag-run/dag-directions)]]]
+ (api/read-check :model/Transform id)
+ (mapv (fn [{xform-id :id, xform-name :name}]
+ {:id xform-id, :name xform-name})
+ (transforms.core/dag-run-transforms id direction)))
+
(def ^{:arglists '([request respond raise])} routes
"`/api/transform` routes."
(handlers/routes
(api.macros/ns-handler *ns* +auth)))
+(def ^{:arglists '([request respond raise])} transform-dag-run-routes
+ "`/api/transform-dag-run` routes."
+ (api.macros/ns-handler 'metabase.transforms-rest.api.transform-dag-run +auth))
+
(def ^{:arglists '([request respond raise])} transform-tag-routes
"`/api/transform-tag` routes."
(api.macros/ns-handler 'metabase.transforms-rest.api.transform-tag +auth))
diff --git a/src/metabase/transforms_rest/api/transform_dag_run.clj b/src/metabase/transforms_rest/api/transform_dag_run.clj
new file mode 100644
index 00000000000..c79c0268fb2
--- /dev/null
+++ b/src/metabase/transforms_rest/api/transform_dag_run.clj
@@ -0,0 +1,49 @@
+(ns metabase.transforms-rest.api.transform-dag-run
+ "`/api/transform-dag-run` routes: operations on a single manual DAG-reprocess run — listing its
+ member transform runs and canceling it. Triggering and previewing a DAG run live on
+ `/api/transform/:id/...` in [[metabase.transforms-rest.api.transform]]."
+ (:require
+ [metabase.api.common :as api]
+ [metabase.api.macros :as api.macros]
+ [metabase.api.routes.common :refer [+auth]]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms-rest.api.util :as transforms-rest.api.u]
+ [metabase.transforms.core :as transforms.core]
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def dag-directions
+ "The DAG traversal directions a reprocess run can take."
+ [:upstream :downstream])
+
+(defn- check-seed-transform
+ "Check access to a DAG run by applying `check!` (`api/read-check`/`api/write-check`) to its seed
+ transform. When the seed was deleted (`source_transform_id` nulled out by the FK) there is no
+ transform left to check against, so fall back to requiring the data-analyst role."
+ [check! {:keys [source_transform_id] :as _dag-run}]
+ (if source_transform_id
+ (check! :model/Transform source_transform_id)
+ (api/check-data-analyst)))
+
+(api.macros/defendpoint :get "/:run-id/transform-runs" :- [:sequential transforms-rest.api.u/MemberTransformRunResponse]
+ "Get the transform runs that made up a specific DAG run."
+ [{:keys [run-id]} :- [:map [:run-id ms/PositiveInt]]]
+ (let [dag-run (api/check-404 (t2/select-one :model/TransformDagRun :id run-id))]
+ (check-seed-transform api/read-check dag-run)
+ (->> (t2/hydrate (transforms.core/transform-runs-for-dag-run run-id)
+ [:transform :collection :transform_tag_ids])
+ (map transforms-base.u/present-run))))
+
+(api.macros/defendpoint :post "/:run-id/cancel" :- :nil
+ "Cancel an in-progress manual DAG run and request cancellation of its still-running transforms."
+ [{:keys [run-id]} :- [:map [:run-id ms/PositiveInt]]]
+ (let [dag-run (api/check-404 (t2/select-one :model/TransformDagRun :id run-id))]
+ (check-seed-transform api/write-check dag-run)
+ (api/check-400 (transforms.core/cancel-dag-run! run-id))
+ nil))
+
+(def ^{:arglists '([request respond raise])} routes
+ "`/api/transform-dag-run` routes."
+ (api.macros/ns-handler *ns* +auth))
diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.clj
index 82ab09d0462..9270b12c3d2 100644
--- a/src/metabase/transforms_rest/api/transform_job.clj
+++ b/src/metabase/transforms_rest/api/transform_job.clj
@@ -7,10 +7,10 @@
[metabase.models.interface :as mi]
[metabase.request.core :as request]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms-rest.api.util :as transforms-rest.api.u]
[metabase.transforms.core :as transforms.core]
[metabase.transforms.util :as transforms.u]
[metabase.util.i18n :refer [deferred-tru LocalizedString]]
- [metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -24,8 +24,10 @@
[:map {:closed true}
[:id pos-int?]
[:job_id pos-int?]
+ [:job_name {:optional true} [:maybe :string]]
+ [:job_entity_id {:optional true} [:maybe :string]]
[:run_method :keyword]
- [:status [:enum :started :succeeded :failed :timeout]]
+ [:status [:enum :started :succeeded :failed :timeout :canceled]]
[:is_active [:maybe :boolean]]
[:start_time :any]
[:end_time {:optional true} [:maybe :any]]
@@ -216,27 +218,28 @@
(transforms.core/delete-job! job-id)
api/generic-204-no-content)
-(api.macros/defendpoint :post "/:job-id/run" :- [:map {:closed true}
- [:message :string]
- [:job_run_id :string]]
- "Run a transform job manually. By default, fresh pulled-in dependencies are skipped; pass `run_all`
- to force-refresh the whole plan."
+(api.macros/defendpoint :post "/:job-id/run" :- [:map
+ [:status [:= 202]]
+ [:body [:map {:closed true}
+ [:message :any]
+ [:job_run_id [:maybe pos-int?]]]]]
+ "Run a transform job manually. Returns a 202 with the created `job_run_id`, or a nil `job_run_id`
+ when nothing was run (the job is already running, or has no transforms). By default, fresh
+ pulled-in dependencies are skipped; pass `run_all` to force-refresh the whole plan."
[{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]
_query-params
{:keys [run_all]} :- [:map
[:run_all {:default false} :boolean]]]
(log/info "Manual run of transform job" job-id)
(api/write-check (t2/select-one :model/TransformJob :id job-id))
- (u.jvm/in-virtual-thread*
- (try
+ (transforms-rest.api.u/async-run-response
+ (deferred-tru "Job run started")
+ :job_run_id
+ (fn [start-promise]
(transforms.core/run-job! job-id {:run-method :manual
:user-id api/*current-user-id*
- :skip-fresh-deps? (not run_all)})
- (catch Throwable t
- (log/error "Error executing transform job" job-id)
- (log/error t))))
- {:message "Job run started"
- :job_run_id (str "stub-" job-id "-" (System/currentTimeMillis))})
+ :skip-fresh-deps? (not run_all)
+ :start-promise start-promise}))))
(api.macros/defendpoint :get "/:job-id" :- TransformJobResponse
"Get a transform job by ID."
@@ -290,8 +293,10 @@
[:map {:closed true}
[:id pos-int?]
[:job_id pos-int?]
+ [:job_name {:optional true} [:maybe :string]]
+ [:job_entity_id {:optional true} [:maybe :string]]
[:run_method :keyword]
- [:status [:enum :started :succeeded :failed :timeout]]
+ [:status [:enum :started :succeeded :failed :timeout :canceled]]
[:is_active [:maybe :boolean]]
[:start_time :any]
[:end_time {:optional true} [:maybe :any]]
@@ -299,26 +304,6 @@
[:created_at :any]
[:updated_at :any]])
-(def ^:private TransformRunForJobRunResponse
- [:map {:closed true}
- [:id pos-int?]
- [:transform_id [:maybe pos-int?]]
- [:job_run_id [:maybe pos-int?]]
- [:run_method :keyword]
- [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]
- [:is_active [:maybe :boolean]]
- [:start_time :any]
- [:end_time {:optional true} [:maybe :any]]
- [:message [:maybe :string]]
- [:user_id [:maybe pos-int?]]
- [:transform_name {:optional true} [:maybe :string]]
- [:transform_entity_id {:optional true} [:maybe :string]]
- [:transform {:optional true} [:maybe :map]]
- [:metered_as {:optional true} [:maybe :string]]
- [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
- [:checkpoint_lo_value {:optional true} [:maybe :string]]
- [:checkpoint_hi_value {:optional true} [:maybe :string]]])
-
(api.macros/defendpoint :get "/:job-id/runs" :- [:map {:closed true}
[:data [:sequential JobRunResponse]]
[:limit pos-int?]
@@ -327,7 +312,7 @@
"Get paginated run history for a transform job."
[{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]
query-params :- [:map
- [:status {:optional true} [:maybe [:enum "started" "succeeded" "failed" "timeout"]]]
+ [:status {:optional true} [:maybe [:enum "started" "succeeded" "failed" "timeout" "canceled"]]]
[:run-method {:optional true} [:maybe [:enum "manual" "cron"]]]
[:start-time {:optional true} [:maybe ms/NonBlankString]]
[:sort-column {:optional true} [:maybe [:enum "start_time" "end_time"]]]
@@ -339,7 +324,7 @@
:limit (request/limit)))
(update :data #(map transforms-base.u/present-run %))))
-(api.macros/defendpoint :get "/:job-id/runs/:run-id/transform-runs" :- [:sequential TransformRunForJobRunResponse]
+(api.macros/defendpoint :get "/:job-id/runs/:run-id/transform-runs" :- [:sequential transforms-rest.api.u/MemberTransformRunResponse]
"Get the transform runs that made up a specific job run."
[{:keys [job-id run-id]} :- [:map
[:job-id ms/PositiveInt]
@@ -351,6 +336,16 @@
(->> (t2/hydrate runs [:transform :collection :transform_tag_ids])
(map transforms-base.u/present-run))))
+(api.macros/defendpoint :post "/:job-id/runs/:run-id/cancel" :- :nil
+ "Cancel an in-progress job run and request cancellation of its still-running transforms."
+ [{:keys [job-id run-id]} :- [:map
+ [:job-id ms/PositiveInt]
+ [:run-id ms/PositiveInt]]]
+ (api/write-check :model/TransformJob job-id)
+ (api/check-404 (t2/select-one :model/TransformJobRun :id run-id :job_id job-id))
+ (api/check-400 (transforms.core/cancel-job-run! run-id))
+ nil)
+
(def ^{:arglists '([request respond raise])} routes
"`/api/transform-job` routes."
(api.macros/ns-handler *ns* +auth))
diff --git a/src/metabase/transforms_rest/api/util.clj b/src/metabase/transforms_rest/api/util.clj
new file mode 100644
index 00000000000..6c6222f7d62
--- /dev/null
+++ b/src/metabase/transforms_rest/api/util.clj
@@ -0,0 +1,52 @@
+(ns metabase.transforms-rest.api.util
+ "Helpers shared by the transforms REST API namespaces."
+ (:require
+ [metabase.util.jvm :as u.jvm]
+ [metabase.util.log :as log]
+ [ring.util.response :as response]))
+
+(set! *warn-on-reflection* true)
+
+(def MemberTransformRunResponse
+ "Response schema for a member transform run of a coordinated (job or DAG) run, hydrated with its
+ transform. Exactly one of `job_run_id`/`dag_run_id` is set, naming the coordinating run."
+ [:map {:closed true}
+ [:id pos-int?]
+ [:transform_id [:maybe pos-int?]]
+ [:job_run_id [:maybe pos-int?]]
+ [:dag_run_id [:maybe pos-int?]]
+ [:run_method :keyword]
+ [:status [:enum :started :succeeded :failed :timeout :canceled :canceling]]
+ [:is_active [:maybe :boolean]]
+ [:start_time :any]
+ [:end_time {:optional true} [:maybe :any]]
+ [:message [:maybe :string]]
+ [:user_id [:maybe pos-int?]]
+ [:transform_name {:optional true} [:maybe :string]]
+ [:transform_entity_id {:optional true} [:maybe :string]]
+ [:transform {:optional true} [:maybe :map]]
+ [:metered_as {:optional true} [:maybe :string]]
+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
+ [:checkpoint_lo_value {:optional true} [:maybe :string]]
+ [:checkpoint_hi_value {:optional true} [:maybe :string]]])
+
+(defn async-run-response
+ "Launch `start!` — a fn of a start-promise — on a virtual thread and respond `202` as soon as the
+ run row exists, without waiting for the run to finish. `start!` must deliver the promise
+ `[:started run-id]` once the row is created, `nil` when nothing was run, or a Throwable on a
+ pre-start failure (rethrown). The response body is `{:message message, <id-key> <run-id-or-nil>}`."
+ [message id-key start!]
+ (let [start-promise (promise)]
+ (u.jvm/in-virtual-thread*
+ (try
+ (start! start-promise)
+ (catch Throwable t
+ ;; post-start failures: the run row was already failed by the runner; this is a fallback log
+ (log/error t "Async run failed"))))
+ (when (instance? Throwable @start-promise)
+ (throw @start-promise))
+ (let [result @start-promise
+ run-id (when (and (vector? result) (= (first result) :started))
+ (second result))]
+ (-> (response/response {:message message, id-key run-id})
+ (assoc :status 202)))))
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 64fe7e1e734..afee9df4fed 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -74,6 +74,7 @@
:model/Transform
:model/TransformRun
:model/TransformRunCancelation
+ :model/TransformDagRun
:model/TransformJob
:model/TransformJobRun
:model/TransformJobTransformTag
diff --git a/test/metabase/transforms/dag_test.clj b/test/metabase/transforms/dag_test.clj
new file mode 100644
index 00000000000..821470d5e35
--- /dev/null
+++ b/test/metabase/transforms/dag_test.clj
@@ -0,0 +1,161 @@
+(ns metabase.transforms.dag-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [metabase.transforms.coordinated-run :as coordinated-run]
+ [metabase.transforms.dag :as dag]
+ [metabase.transforms.models.dag-run :as dag-run]
+ [metabase.transforms.models.transform-run :as transform-run]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------- Pure graph logic -------------------------------------------
+
+(deftest dependents-graph-test
+ (testing "reverses a forward dependency map (reads) into a dependents map (read-by)"
+ (is (= {1 #{2 3} 2 #{4} 3 #{4}}
+ (#'dag/dependents-graph {2 #{1} 3 #{1} 4 #{2 3} 1 #{}}))))
+ (testing "a node nothing depends on is absent from the dependents map"
+ (is (= {1 #{2}}
+ (#'dag/dependents-graph {2 #{1}}))))
+ (testing "empty graph"
+ (is (= {} (#'dag/dependents-graph {})))))
+
+(deftest reachable-test
+ (let [graph {1 #{2 3} 2 #{4} 3 #{4} 4 #{}}]
+ (testing "includes the start node and everything transitively reachable"
+ (is (= #{1 2 3 4} (#'dag/reachable graph 1)))
+ (is (= #{2 4} (#'dag/reachable graph 2)))
+ (is (= #{4} (#'dag/reachable graph 4))))
+ (testing "a start node with no outgoing edges reaches only itself"
+ (is (= #{5} (#'dag/reachable {} 5)))))
+ (testing "cycles terminate (each node visited once)"
+ (is (= #{1 2 3} (#'dag/reachable {1 #{2} 2 #{3} 3 #{1}} 1)))))
+
+;;; ------------------------------------------- run-dag! guard -------------------------------------------
+
+(deftest run-dag-skips-when-already-running-test
+ (testing "run-dag! does not start a second run while one is already active for the seed transform"
+ (let [start-promise (promise)
+ planned? (atom false)]
+ (mt/with-dynamic-fn-redefs [dag-run/running-run-for-source-transform-id (constantly {:id 1})
+ dag-run/start-dag-run! (fn [& _] (reset! planned? true) {:id 2})]
+ (is (nil? (dag/run-dag! 42 {:direction :upstream :start-promise start-promise})))
+ (is (false? @planned?) "should not create a DAG run row when one is already running")
+ (is (nil? (deref start-promise 100 :timed-out))
+ "start-promise is delivered nil so a waiting caller doesn't hang")))))
+
+;; test-helpers-set-global-values!: these tests trigger an intentional duplicate-key violation,
+;; which on Postgres would abort (poison) the transaction with-temp otherwise wraps the body in
+(deftest start-dag-run-unique-active-run-test
+ (testing "the (source_transform_id, is_active) unique constraint allows at most one active run per seed"
+ (mt/test-helpers-set-global-values!
+ (mt/with-temp [:model/Transform {tid :id} {:name "seed"}]
+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]
+ (try
+ (is (thrown? Exception (dag-run/start-dag-run! tid :downstream nil 1)))
+ (testing "a terminal run doesn't collide, so a new run can start"
+ (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)
+ (let [{run-id-2 :id} (dag-run/start-dag-run! tid :downstream nil 1)]
+ (is (pos-int? run-id-2))))
+ (finally
+ (t2/delete! :model/TransformDagRun :source_transform_id tid))))))))
+
+(deftest run-dag-start-race-test
+ (testing "run-dag! treats losing the unique-active-run insert race as already-running"
+ (mt/test-helpers-set-global-values!
+ (mt/with-temp [:model/Transform {tid :id} {:name "seed" :table_dependencies []}]
+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)
+ start-promise (promise)]
+ (try
+ ;; bypass the fast pre-check to simulate two triggers racing past it together; the loser's
+ ;; insert then hits the unique constraint and must be treated as "already running"
+ (mt/with-dynamic-fn-redefs [dag-run/running-run-for-source-transform-id (constantly nil)]
+ (is (nil? (dag/run-dag! tid {:direction :upstream :start-promise start-promise})))
+ (is (nil? (deref start-promise 100 :timed-out))
+ "start-promise is delivered nil so a waiting caller doesn't hang"))
+ (finally
+ (t2/delete! :model/TransformDagRun :id run-id))))))))
+
+;;; ------------------------------------------- Model lifecycle -------------------------------------------
+
+(deftest dag-run-lifecycle-test
+ (mt/with-temp [:model/Transform {tid :id} {:name "seed"}]
+ (testing "start-dag-run! creates a started, active run for the seed transform"
+ (let [{run-id :id :as run} (dag-run/start-dag-run! tid :upstream (mt/user->id :rasta) 3)]
+ (try
+ (is (= :started (:status run)))
+ (is (true? (:is_active run)))
+ (is (= tid (:source_transform_id run)))
+ (testing "the seed transform's name/entity_id are snapshotted for post-deletion display"
+ (is (= "seed" (:source_transform_name run)))
+ (is (some? (:source_transform_entity_id run))))
+ (is (= 3 (:transform_count run)))
+ (is (= :upstream (:direction run)))
+ (is (= (mt/user->id :rasta) (:user_id run)))
+ (testing "running-run-for-source-transform-id finds the active run"
+ (is (= run-id (:id (dag-run/running-run-for-source-transform-id tid)))))
+ (testing "succeed-started-run! makes it terminal and inactive"
+ (is (= 1 (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)))
+ (is (= :succeeded (:status (t2/select-one :model/TransformDagRun :id run-id))))
+ (is (nil? (:is_active (t2/select-one :model/TransformDagRun :id run-id))))
+ (is (nil? (dag-run/running-run-for-source-transform-id tid))))
+ (finally
+ (t2/delete! :model/TransformDagRun :id run-id)))))
+ (testing "cancel-started-run! cancels an active run but is a no-op once terminal"
+ (let [{run-id :id} (dag-run/start-dag-run! tid :downstream nil 1)]
+ (try
+ (is (= 1 (coordinated-run/cancel-started-run! :model/TransformDagRun run-id)))
+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))
+ (is (zero? (coordinated-run/cancel-started-run! :model/TransformDagRun run-id))
+ "a finished run is never resurrected into a canceled state")
+ (finally
+ (t2/delete! :model/TransformDagRun :id run-id)))))
+ (testing "fail-started-run! records the failure message"
+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]
+ (try
+ (coordinated-run/fail-started-run! :model/TransformDagRun run-id {:message "boom"})
+ (let [row (t2/select-one :model/TransformDagRun :id run-id)]
+ (is (= :failed (:status row)))
+ (is (= "boom" (:message row))))
+ (finally
+ (t2/delete! :model/TransformDagRun :id run-id)))))))
+
+(deftest dag-run-survives-transform-deletion-test
+ (testing "deleting the seed transform nulls source_transform_id but preserves the DAG run and its snapshot"
+ (mt/with-temp [:model/Transform {tid :id} {:name "doomed seed"}]
+ (let [{run-id :id} (dag-run/start-dag-run! tid :upstream nil 1)]
+ (try
+ (coordinated-run/succeed-started-run! :model/TransformDagRun run-id)
+ (t2/delete! :model/Transform :id tid)
+ (let [run (t2/select-one :model/TransformDagRun :id run-id)]
+ (is (some? run) "the run row survives (FK is SET NULL, not CASCADE)")
+ (is (nil? (:source_transform_id run)))
+ (is (= "doomed seed" (:source_transform_name run))))
+ (finally
+ (t2/delete! :model/TransformDagRun :id run-id)))))))
+
+(deftest dag-run-member-runs-test
+ ;; two distinct member transforms: transform_run enforces one active run per transform
+ (mt/with-temp [:model/Transform {seed :id} {:name "seed"}
+ :model/Transform {tid-a :id} {:name "member-a"}
+ :model/Transform {tid-b :id} {:name "member-b"}]
+ (let [{run-id :id} (dag-run/start-dag-run! seed :upstream nil 2)]
+ (try
+ (let [{active-run :id} (transform-run/start-run! tid-a {:dag_run_id run-id :run_method :manual})
+ {done-run :id} (transform-run/start-run! tid-b {:dag_run_id run-id :run_method :manual})]
+ (transform-run/succeed-started-run! done-run)
+ (testing "transform-runs-for-dag-run returns all member runs"
+ (is (= #{active-run done-run}
+ (set (map :id (dag-run/transform-runs-for-dag-run run-id))))))
+ (testing "cancel! cancels the run and requests cancelation of only the still-active members"
+ (is (true? (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id)))
+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))
+ (is (t2/exists? :model/TransformRunCancelation :run_id active-run))
+ (is (not (t2/exists? :model/TransformRunCancelation :run_id done-run)))
+ (is (false? (coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id))
+ "canceling an already-terminal run is a no-op")))
+ (finally
+ (t2/delete! :model/TransformRun :dag_run_id run-id)
+ (t2/delete! :model/TransformDagRun :id run-id))))))
diff --git a/test/metabase/transforms/heartbeat_test.clj b/test/metabase/transforms/heartbeat_test.clj
index 1a1bcb89e3b..6370a7048c1 100644
--- a/test/metabase/transforms/heartbeat_test.clj
+++ b/test/metabase/transforms/heartbeat_test.clj
@@ -6,6 +6,7 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.transforms.canceling :as canceling]
+ [metabase.transforms.coordinated-run :as coordinated-run]
[metabase.transforms.models.job-run :as job-run]
[metabase.transforms.models.transform-run :as transform-run]
[toucan2.core :as t2])
@@ -143,14 +144,14 @@
:is_active nil
:run_method :manual
:last_heartbeat (minutes-ago 10)}]
- (job-run/heartbeat-runs! [beat-id done-id])
+ (coordinated-run/heartbeat-runs! :model/TransformJobRun [beat-id done-id])
(testing "only the passed run that is still active gets a fresh heartbeat"
(is (job-recently-beaten? beat-id))
(is (not (job-recently-beaten? other-id)) "a run not in the id list is left to go stale"))
(testing "an inactive run is not stamped even when passed (is_active guard)"
(is (not (job-recently-beaten? done-id))))
(testing "empty id list is a no-op"
- (is (nil? (job-run/heartbeat-runs! [])))))))
+ (is (nil? (coordinated-run/heartbeat-runs! :model/TransformJobRun [])))))))
(deftest job-reap-orphaned-runs!-test
(mt/with-premium-features #{:transforms-basic}
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index 8c50c564214..194ca81f8e3 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -141,9 +141,10 @@
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
- (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)
+ (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
(is @run-called?
@@ -159,7 +160,9 @@
(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)]
- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)
+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) query-transform)
(is (= 1 (count @logged-messages))
"Should log exactly one warning")
(is (= :warn (:level (first @logged-messages)))
@@ -180,9 +183,10 @@
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
- (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)
+ (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
(is @run-called?
@@ -201,9 +205,10 @@
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
- (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! run-id :scheduled nil (promise) query-transform)
+ (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) query-transform)
(is (= 1 (count @logged-messages))
"Should log exactly one warning")
(is (= :warn (:level (first @logged-messages)))
@@ -236,9 +241,10 @@
(mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
(swap! logged conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/execute! (fn [_ _] (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! 200 :scheduled nil (promise) transform)
+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job 200] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) transform)
(is (false? @run-called?)
"execute! must not be called when the meter is locked")
(is (some #(re-matches #".*Skip running transform 7 due to locked meter.*"
@@ -251,9 +257,10 @@
(let [transform {:id 8 :source_type :native :source query-source :name "Unlocked"}
run-called? (atom false)]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/execute! (fn [_ _] (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! 201 :scheduled nil (promise) transform)
+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job 201] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) transform)
(is (true? @run-called?)))))))
(testing "non-metered transform (transform-metered-as → nil) is never blocked by lock state"
;; Override the outer mock with one that returns nil for every source-type.
@@ -264,9 +271,10 @@
(let [transform {:id 9 :source_type :native :source query-source :name "Non-metered"}
run-called? (atom false)]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/execute! (fn [_ _] (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
- (#'jobs/run-transform! 202 :scheduled nil (promise) transform)
+ transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
+ (#'jobs/run-transform! {:parent-run [:job 202] :run-method :scheduled :user-id nil
+ :add-run-activity! (constantly nil)}
+ (promise) transform)
(is (true? @run-called?)
"Non-metered transforms are not gated by lock state")))))))))
@@ -514,30 +522,28 @@
deps {1 #{} 2 #{1} 3 #{}}
ran (atom #{})]
(testing "a fresh pulled-in dep is not executed, but its dependent still runs"
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- freshness/fresh-dep-ids (fn [_now dep-ids]
+ (with-redefs [freshness/fresh-dep-ids (fn [_now dep-ids]
;; only pulled-in deps are offered for gating
(is (= #{1} (set dep-ids)))
#{1})
- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]
+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{2 3} {:run-method :cron})))
+ (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps} {:run-method :cron})))
(is (= #{2 3} @ran) "transform 1 (fresh dep) was skipped; 2 and 3 ran")))
(testing "skip-fresh-deps? false (manual run_all) force-refreshes the whole plan"
(reset! ran #{})
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- freshness/fresh-dep-ids (fn [& _] (throw (ex-info "must not gate when disabled" {})))
- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]
+ (with-redefs [freshness/fresh-dep-ids (fn [& _] (throw (ex-info "must not gate when disabled" {})))
+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{2 3} {:run-method :manual :skip-fresh-deps? false})))
+ (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps}
+ {:run-method :manual :skip-fresh-deps? false})))
(is (= #{1 2 3} @ran) "with skipping disabled, the dep runs too"))))
(testing "a plan that is entirely skipped exits cleanly as succeeded with nothing run"
(let [ran (atom #{})]
- (with-redefs [jobs/get-plan (fn [_] {:order [{:id 1}] :deps {1 #{}}})
- freshness/fresh-dep-ids (fn [_now _dep-ids] #{1})
- jobs/run-transform! (fn [_ _ _ _ t] (swap! ran conj (:id t)))]
+ (with-redefs [freshness/fresh-dep-ids (fn [_now _dep-ids] #{1})
+ jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{} {:run-method :cron})))
+ (jobs/run-transforms! 0 #{} {:order [{:id 1}] :deps {1 #{}}} {:run-method :cron})))
(is (= #{} @ran)))))))
(deftest run-transforms!-race-condition-test
@@ -594,17 +600,18 @@
(catch Throwable t [nil t]))]
(on-exit)
(if ex (throw ex) ret)))]
- (let [run1 (transforms.job-run/start-run! (:id job) :manual)
+ (let [plan (#'jobs/get-plan #{(:id transform)})
+ run1 (transforms.job-run/start-run! (:id job) :manual)
run2 (transforms.job-run/start-run! (:id job) :manual)
fut1 (future
(try
- {:result (jobs/run-transforms! (:id run1) #{(:id transform)}
+ {:result (jobs/run-transforms! (:id run1) #{(:id transform)} plan
{:run-method :manual})}
(catch Exception e
{:error e})))
fut2 (future
(try
- {:result (jobs/run-transforms! (:id run2) #{(:id transform)}
+ {:result (jobs/run-transforms! (:id run2) #{(:id transform)} plan
{:run-method :manual})}
(catch Exception e
{:error e})))
@@ -622,33 +629,30 @@
deps {1 #{} 2 #{} 3 #{} 4 #{}}
barrier (CyclicBarrier. 4)]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ _]
+ (with-redefs [jobs/run-transform! (fn [_ _ _]
(.await barrier 5 TimeUnit/SECONDS))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual}))
+ (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual}))
"all four independents must reach the barrier (i.e. be live simultaneously)")))))
(testing "Dependents wait for their dependencies even when concurrency > 1"
(let [plan [{:id 1} {:id 2}]
deps {1 #{} 2 #{1}}
order (atom [])]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ transform]
+ (with-redefs [jobs/run-transform! (fn [_ _ transform]
(when (= 1 (:id transform))
(Thread/sleep 100))
(swap! order conj (:id transform)))]
- (jobs/run-transforms! 0 #{1 2} {:run-method :manual})
+ (jobs/run-transforms! 0 #{1 2} {:order plan :deps deps} {:run-method :manual})
(is (= [1 2] @order) "a must complete before b starts")))))
(testing "Dependents of a failed transform are skipped transitively"
(let [plan [{:id 1} {:id 2} {:id 3}]
deps {1 #{} 2 #{1} 3 #{2}}]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ transform]
+ (with-redefs [jobs/run-transform! (fn [_ _ transform]
(when (= 1 (:id transform))
(throw (ex-info "boom" {}))))]
- (let [result (jobs/run-transforms! 0 #{1 2 3} {:run-method :manual})]
+ (let [result (jobs/run-transforms! 0 #{1 2 3} {:order plan :deps deps} {:run-method :manual})]
(is (= :failed (::jobs/status result)))
(is (= #{1 2 3} (set (map (comp :id ::jobs/transform) (::jobs/failures result))))
"a failed, b & c skipped as dep-failed"))))))
@@ -661,13 +665,12 @@
live (atom 0)
max-live (atom 0)]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 2]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ _]
+ (with-redefs [jobs/run-transform! (fn [_ _ _]
(let [n (swap! live inc)]
(swap! max-live max n))
(.await partner-up 5 TimeUnit/SECONDS)
(swap! live dec))]
- (jobs/run-transforms! 0 (set (range 1 9)) {:run-method :manual})
+ (jobs/run-transforms! 0 (set (range 1 9)) {:order plan :deps deps} {:run-method :manual})
(is (= 2 @max-live) "should never exceed the concurrency setting")))))
(testing "Python transforms are serialized in their own lane (n=1)"
(let [py (fn [id] {:id id :source {:type :python}})
@@ -676,15 +679,14 @@
live (atom 0)
max-live (atom 0)]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ _]
+ (with-redefs [jobs/run-transform! (fn [_ _ _]
(let [n (swap! live inc)]
(swap! max-live max n))
;; A short sleep gives any (hypothetical) parallel
;; python worker a window to show up in the counter.
(Thread/sleep 50)
(swap! live dec))]
- (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual})
+ (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual})
(is (= 1 @max-live)
"python transforms must run one at a time even with concurrency=4")))))
(testing "SQL and Python lanes run in parallel without contending for slots"
@@ -696,8 +698,7 @@
py-live (atom 0)
max-py (atom 0)]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 3]
- (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
- jobs/run-transform! (fn [_ _ _ _ transform]
+ (with-redefs [jobs/run-transform! (fn [_ _ transform]
(if (= :python (-> transform :source :type))
(do (let [n (swap! py-live inc)]
(swap! max-py max n))
@@ -705,6 +706,6 @@
(swap! py-live dec))
(.await sql-barrier 5 TimeUnit/SECONDS)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{1 2 3 4 5} {:run-method :manual}))
+ (jobs/run-transforms! 0 #{1 2 3 4 5} {:order plan :deps deps} {:run-method :manual}))
"all three SQL transforms reach the barrier (run in parallel)")
(is (= 1 @max-py) "python lane stays single-slot")))))))
diff --git a/test/metabase/transforms_rest/api/transform_dag_run_test.clj b/test/metabase/transforms_rest/api/transform_dag_run_test.clj
new file mode 100644
index 00000000000..196f3edc0c9
--- /dev/null
+++ b/test/metabase/transforms_rest/api/transform_dag_run_test.clj
@@ -0,0 +1,245 @@
+(ns metabase.transforms-rest.api.transform-dag-run-test
+ "Tests for the /api/transform-dag-run endpoints, plus the per-transform DAG endpoints on
+ /api/transform/:id (previewing and triggering a DAG-reprocess run)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.test :as mt]
+ [metabase.transforms.test-util :refer [parse-instant utc-timestamp]]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; transforms-enabled: transform read/write perms are gated on the transforms kill switch, which
+;; defaults to off outside cloud — without it every endpoint here 403s regardless of permissions
+
+;;; +------------------------------------------------------------------------------------------------------------------+
+;;; | GET /api/transform-dag-run/:run-id/transform-runs |
+;;; +------------------------------------------------------------------------------------------------------------------+
+
+(deftest get-dag-run-transform-runs-test
+ (testing "GET /api/transform-dag-run/:run-id/transform-runs"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}
+ :model/Transform {t1-id :id} {:name "TR1"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id
+ :direction "upstream"
+ :status "succeeded"
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/TransformRun {tr1-id :id} {:transform_id t1-id
+ :dag_run_id run-id
+ :status "succeeded"
+ :run_method "manual"
+ :transform_name "TR1"
+ :start_time (parse-instant "2025-09-01T10:00:00")
+ :end_time (parse-instant "2025-09-01T10:02:00")}
+ :model/TransformRun {tr2-id :id} {:transform_id seed-id
+ :dag_run_id run-id
+ :status "failed"
+ :run_method "manual"
+ :transform_name "Seed"
+ :start_time (parse-instant "2025-09-01T10:02:00")
+ :message "Query failed"}
+ ;; a run of the same transform outside the DAG run must not leak in
+ :model/TransformRun _ {:transform_id t1-id
+ :status "succeeded"
+ :run_method "manual"
+ :transform_name "TR1"
+ :start_time (parse-instant "2025-08-01T10:00:00")}]
+ (testing "returns the member runs, ordered by start_time ascending"
+ (let [response (mt/user-http-request :lucky :get 200
+ (str "transform-dag-run/" run-id "/transform-runs"))]
+ (is (= [tr1-id tr2-id] (map :id response)))
+ (is (every? #(= run-id (:dag_run_id %)) response))))
+ (testing "response shape"
+ (let [response (mt/user-http-request :lucky :get 200
+ (str "transform-dag-run/" run-id "/transform-runs"))
+ tr1 (first (filter #(= tr1-id (:id %)) response))]
+ (is (= t1-id (:transform_id tr1)))
+ (is (= "manual" (:run_method tr1)))
+ (is (= "succeeded" (:status tr1)))
+ (is (= "TR1" (:transform_name tr1)))
+ (is (= (utc-timestamp "2025-09-01T10:00:00") (:start_time tr1)))
+ (is (= (utc-timestamp "2025-09-01T10:02:00") (:end_time tr1)))))
+ (testing "returns 404 for a non-existent DAG run"
+ (mt/user-http-request :lucky :get 404 "transform-dag-run/999999/transform-runs"))))))))
+
+(deftest get-dag-run-transform-runs-deleted-seed-test
+ (testing "GET /api/transform-dag-run/:run-id/transform-runs still lists member runs after the seed transform is deleted"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ ;; source_transform_id nil as the SET NULL FK leaves it after the seed transform is deleted
+ (mt/with-temp [:model/Transform {t1-id :id} {:name "TR1"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id nil
+ :source_transform_name "Deleted Seed"
+ :direction "upstream"
+ :status "succeeded"
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/TransformRun {tr1-id :id} {:transform_id t1-id
+ :dag_run_id run-id
+ :status "succeeded"
+ :run_method "manual"
+ :transform_name "TR1"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (let [response (mt/user-http-request :lucky :get 200 (str "transform-dag-run/" run-id "/transform-runs"))]
+ (is (= [tr1-id] (map :id response))))
+ (testing "non-data-analysts are still denied"
+ (mt/user-http-request :rasta :get 403 (str "transform-dag-run/" run-id "/transform-runs")))))))))
+
+(deftest get-dag-run-transform-runs-requires-read-permission-test
+ (testing "GET /api/transform-dag-run/:run-id/transform-runs requires read permission on the seed transform"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id
+ :direction "upstream"
+ :status "succeeded"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :rasta :get 403 (str "transform-dag-run/" run-id "/transform-runs")))))))
+
+;;; +------------------------------------------------------------------------------------------------------------------+
+;;; | POST /api/transform-dag-run/:run-id/cancel |
+;;; +------------------------------------------------------------------------------------------------------------------+
+
+(deftest cancel-dag-run-test
+ (testing "POST /api/transform-dag-run/:run-id/cancel"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ ;; canceling write-checks the seed transform, which needs the transforms permission on its source DB
+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
+ (testing "cancels an active DAG run and requests cancellation of its active members"
+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}
+ :model/Transform {t1-id :id} {:name "TR1"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id
+ :direction "upstream"
+ :status "started"
+ :is_active true
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/TransformRun {member-id :id} {:transform_id t1-id
+ :dag_run_id run-id
+ :status "started"
+ :is_active true
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :lucky :post 204 (str "transform-dag-run/" run-id "/cancel"))
+ (let [run (t2/select-one :model/TransformDagRun :id run-id)]
+ (is (= :canceled (:status run)))
+ (is (nil? (:is_active run)))
+ (is (some? (:end_time run))))
+ (testing "a cancelation row is recorded for the still-running member"
+ (is (t2/exists? :model/TransformRunCancelation :run_id member-id)))))
+ (testing "returns 400 when the run has already finished"
+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id
+ :direction "downstream"
+ :status "succeeded"
+ :start_time (parse-instant "2025-09-01T10:00:00")
+ :end_time (parse-instant "2025-09-01T10:05:00")}]
+ (mt/user-http-request :lucky :post 400 (str "transform-dag-run/" run-id "/cancel"))
+ (is (= :succeeded (:status (t2/select-one :model/TransformDagRun :id run-id)))
+ "a finished run is never resurrected into a canceled state")))
+ (testing "returns 404 for a non-existent DAG run"
+ (mt/user-http-request :lucky :post 404 "transform-dag-run/999999/cancel"))))))))
+
+(deftest cancel-dag-run-deleted-seed-test
+ (testing "POST /api/transform-dag-run/:run-id/cancel still works after the seed transform is deleted"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/TransformDagRun {run-id :id} {:source_transform_id nil
+ :source_transform_name "Deleted Seed"
+ :direction "upstream"
+ :status "started"
+ :is_active true
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :lucky :post 204 (str "transform-dag-run/" run-id "/cancel"))
+ (is (= :canceled (:status (t2/select-one :model/TransformDagRun :id run-id))))))))))
+
+(deftest cancel-dag-run-requires-write-permission-test
+ (testing "POST /api/transform-dag-run/:run-id/cancel requires write permission on the seed transform"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {seed-id :id} {:name "Seed"}
+ :model/TransformDagRun {run-id :id} {:source_transform_id seed-id
+ :direction "upstream"
+ :status "started"
+ :is_active true
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :rasta :post 403 (str "transform-dag-run/" run-id "/cancel")))))))
+
+;;; +------------------------------------------------------------------------------------------------------------------+
+;;; | GET /api/transform/:id/dag-transforms |
+;;; +------------------------------------------------------------------------------------------------------------------+
+
+(deftest dag-transforms-preview-test
+ (testing "GET /api/transform/:id/dag-transforms"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ ;; consumer reads producer's output table via the stored table_dependencies column, so the
+ ;; dependency graph resolves without parsing any queries
+ (mt/with-temp [:model/Transform {producer-id :id} {:name "Producer"
+ :target_table_id (mt/id :orders)
+ :table_dependencies []}
+ :model/Transform {consumer-id :id} {:name "Consumer"
+ :table_dependencies [{:table (mt/id :orders)}]}]
+ (testing "upstream preview returns the dependency closure in execution order"
+ (is (= [{:id producer-id :name "Producer"}
+ {:id consumer-id :name "Consumer"}]
+ (mt/user-http-request :lucky :get 200 (str "transform/" consumer-id "/dag-transforms")
+ :direction "upstream"))))
+ (testing "upstream preview of a transform with no dependencies is just itself"
+ (is (= [{:id producer-id :name "Producer"}]
+ (mt/user-http-request :lucky :get 200 (str "transform/" producer-id "/dag-transforms")
+ :direction "upstream"))))
+ (testing "downstream preview returns the dependents closure in execution order"
+ (is (= [{:id producer-id :name "Producer"}
+ {:id consumer-id :name "Consumer"}]
+ (mt/user-http-request :lucky :get 200 (str "transform/" producer-id "/dag-transforms")
+ :direction "downstream"))))
+ (testing "direction is required and validated"
+ (mt/user-http-request :lucky :get 400 (str "transform/" consumer-id "/dag-transforms"))
+ (mt/user-http-request :lucky :get 400 (str "transform/" consumer-id "/dag-transforms")
+ :direction "sideways"))
+ (testing "returns 404 for a non-existent transform"
+ (mt/user-http-request :lucky :get 404 "transform/999999/dag-transforms"
+ :direction "upstream"))))))))
+
+;;; +------------------------------------------------------------------------------------------------------------------+
+;;; | POST /api/transform/:id/run-dag |
+;;; +------------------------------------------------------------------------------------------------------------------+
+
+(deftest run-dag-already-running-test
+ (testing "POST /api/transform/:id/run-dag returns 202 with a nil dag_run_id when a DAG run is already in progress"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}
+ :model/TransformDagRun _ {:source_transform_id tid
+ :direction "upstream"
+ :status "started"
+ :is_active true
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (let [response (mt/user-http-request :crowberto :post 202 (str "transform/" tid "/run-dag")
+ {:direction "upstream"})]
+ (is (nil? (:dag_run_id response)))))))))
+
+(deftest run-dag-validates-direction-test
+ (testing "POST /api/transform/:id/run-dag requires a valid direction"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}]
+ (mt/user-http-request :crowberto :post 400 (str "transform/" tid "/run-dag") {})
+ (mt/user-http-request :crowberto :post 400 (str "transform/" tid "/run-dag")
+ {:direction "sideways"}))))))
+
+(deftest run-dag-requires-write-permission-test
+ (testing "POST /api/transform/:id/run-dag requires write permission on the transform"
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {tid :id} {:name "Seed"}]
+ (mt/user-http-request :rasta :post 403 (str "transform/" tid "/run-dag")
+ {:direction "upstream"}))))))
diff --git a/test/metabase/transforms_rest/api/transform_job_test.clj b/test/metabase/transforms_rest/api/transform_job_test.clj
index ea96ed1f71f..9c510118537 100644
--- a/test/metabase/transforms_rest/api/transform_job_test.clj
+++ b/test/metabase/transforms_rest/api/transform_job_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase.test :as mt]
+ [metabase.transforms.jobs :as jobs]
[metabase.transforms.models.transform-job]
[metabase.transforms.models.transform-run]
[metabase.transforms.models.transform-tag]
@@ -308,15 +309,42 @@
(mt/user-http-request :lucky :delete 404 "transform-job/999999")))))))
(deftest execute-job-test
- (testing "POST /api/transform-job/:id/execute"
+ (testing "POST /api/transform-job/:id/run"
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-premium-features #{:transforms-basic}
- (mt/with-temp [:model/TransformJob job {:name "To Execute" :schedule "0 0 0 * * ?"}]
- (testing "Returns stub run response"
- (let [response (mt/user-http-request :lucky :post 200 (str "transform-job/" (:id job) "/run"))]
- (is (= "Job run started" (:message response)))
- (is (string? (:job_run_id response)))
- (is (re-matches #"stub-\d+-\d+" (:job_run_id response))))))))))
+ (testing "returns the created run id"
+ (mt/with-model-cleanup [:model/TransformJobRun]
+ (mt/with-temp [:model/TransformTag tag {:name "execute-test-tag"}
+ :model/TransformJob job {:name "To Execute" :schedule "0 0 0 * * ?"}
+ :model/TransformJobTransformTag _ {:job_id (:id job) :tag_id (:id tag) :position 0}
+ :model/Transform transform {:name "To Run"}
+ :model/TransformTransformTag _ {:transform_id (:id transform) :tag_id (:id tag) :position 0}]
+ (let [called (promise)]
+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]
+ (deliver called run-id)
+ {::jobs/status :succeeded})]
+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]
+ (is (= "Job run started" (:message response)))
+ (is (pos-int? (:job_run_id response)))
+ (is (= (:job_run_id response) (deref called 5000 ::timed-out)))
+ (let [run (t2/select-one :model/TransformJobRun :id (:job_run_id response))]
+ (is (= (:id job) (:job_id run)))
+ (testing "the job's name and entity_id are snapshotted on the run"
+ (is (= "To Execute" (:job_name run)))
+ (is (= (:entity_id job) (:job_entity_id run)))))))))))
+ (testing "returns a nil run id when the job has no transforms"
+ (mt/with-temp [:model/TransformJob job {:name "Empty Job" :schedule "0 0 0 * * ?"}]
+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]
+ (is (nil? (:job_run_id response))))))
+ (testing "returns a nil run id when the job is already running"
+ (mt/with-temp [:model/TransformJob job {:name "Busy Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun _ {:job_id (:id job)
+ :status "started"
+ :is_active true
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (let [response (mt/user-http-request :lucky :post 202 (str "transform-job/" (:id job) "/run"))]
+ (is (nil? (:job_run_id response))))))))))
(deftest permissions-test
(testing "All endpoints require transform permissions"
@@ -507,6 +535,70 @@
(mt/user-http-request :rasta :get 403
(str "transform-job/" job-id "/runs/" run-id "/transform-runs"))))))
+;;; +------------------------------------------------------------------------------------------------------------------+
+;;; | POST /api/transform-job/:job-id/runs/:run-id/cancel |
+;;; +------------------------------------------------------------------------------------------------------------------+
+
+(deftest cancel-job-run-test
+ (testing "POST /api/transform-job/:job-id/runs/:run-id/cancel"
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (testing "cancels an active job run and requests cancellation of its active members"
+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {run-id :id} {:job_id job-id
+ :status "started"
+ :is_active true
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/Transform {t1-id :id} {:name "TR1"}
+ :model/TransformRun {member-id :id} {:transform_id t1-id
+ :job_run_id run-id
+ :status "started"
+ :is_active true
+ :run_method "cron"
+ :start_time (parse-instant "2025-09-01T10:00:00")
+ :end_time nil}]
+ (mt/user-http-request :lucky :post 204 (str "transform-job/" job-id "/runs/" run-id "/cancel"))
+ (let [run (t2/select-one :model/TransformJobRun :id run-id)]
+ (is (= :canceled (:status run)))
+ (is (nil? (:is_active run)))
+ (is (some? (:end_time run))))
+ (testing "a cancelation row is recorded for the still-running member"
+ (is (t2/exists? :model/TransformRunCancelation :run_id member-id)))))
+ (testing "returns 400 when the run has already finished"
+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {run-id :id} {:job_id job-id
+ :status "succeeded"
+ :run_method "cron"
+ :start_time (parse-instant "2025-09-01T10:00:00")
+ :end_time (parse-instant "2025-09-01T10:05:00")}]
+ (mt/user-http-request :lucky :post 400 (str "transform-job/" job-id "/runs/" run-id "/cancel"))
+ (is (= :succeeded (:status (t2/select-one :model/TransformJobRun :id run-id)))
+ "a finished run is never resurrected into a canceled state")))
+ (testing "returns 404 for a non-existent job or run, or a run of another job"
+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Cancel Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJob {other-job-id :id} {:name "Other Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {run-id :id} {:job_id job-id
+ :status "started"
+ :is_active true
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :lucky :post 404 (str "transform-job/999999/runs/" run-id "/cancel"))
+ (mt/user-http-request :lucky :post 404 (str "transform-job/" job-id "/runs/999999/cancel"))
+ (mt/user-http-request :lucky :post 404 (str "transform-job/" other-job-id "/runs/" run-id "/cancel"))))))))
+
+(deftest cancel-job-run-requires-write-permission-test
+ (testing "POST /api/transform-job/:job-id/runs/:run-id/cancel requires write permission on the job"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "Auth Test" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {run-id :id} {:job_id job-id
+ :status "started"
+ :is_active true
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-01T10:00:00")}]
+ (mt/user-http-request :rasta :post 403
+ (str "transform-job/" job-id "/runs/" run-id "/cancel"))))))
+
(deftest built-in-jobs-and-tags-seeded-test
(testing "the built-in Hourly/Daily/Weekly/Monthly jobs and tags are pre-seeded (migration data)"
;; Looked up by :name (stable) rather than :entity_id/:built_in_type: editing a built-in job in
diff --git a/test/metabase/transforms_rest/api/transform_runs_test.clj b/test/metabase/transforms_rest/api/transform_runs_test.clj
new file mode 100644
index 00000000000..34d63a99754
--- /dev/null
+++ b/test/metabase/transforms_rest/api/transform_runs_test.clj
@@ -0,0 +1,234 @@
+(ns metabase.transforms-rest.api.transform-runs-test
+ "Tests for GET /api/transform/runs — the unified listing of root runs (job runs, manual
+ DAG-reprocess runs, and standalone transform runs)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [metabase.transforms.test-util :refer [parse-instant utc-timestamp]]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- rows-by-type
+ "Index the response rows by `[run_type id]` so assertions are immune to rows created by other
+ tests (the listing is global)."
+ [response]
+ (into {} (map (juxt (juxt :run_type :id) identity)) (:data response)))
+
+(defn- do-with-runs-fixture!
+ "Call `f` with a map of fixture ids: a job run whose member ran `transform-a`, a DAG run seeded
+ from (and whose member ran) `transform-b`, and a standalone run of `transform-a`."
+ [f]
+ (mt/with-temp [:model/Transform {ta-id :id} {:name "Transform A"}
+ :model/Transform {tb-id :id} {:name "Transform B"}
+ :model/TransformJob {job-id :id} {:name "Nightly Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {job-run-id :id} {:job_id job-id
+ :status "succeeded"
+ :run_method "cron"
+ :start_time (parse-instant "2025-09-01T10:00:00")
+ :end_time (parse-instant "2025-09-01T10:05:00")}
+ :model/TransformRun {member-a-id :id} {:transform_id ta-id
+ :job_run_id job-run-id
+ :status "succeeded"
+ :run_method "cron"
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/TransformDagRun {dag-run-id :id} {:source_transform_id tb-id
+ :direction "downstream"
+ :transform_count 4
+ :status "failed"
+ :message "boom"
+ :user_id (mt/user->id :lucky)
+ :start_time (parse-instant "2025-09-02T10:00:00")
+ :end_time (parse-instant "2025-09-02T10:03:00")}
+ :model/TransformRun {member-b-id :id} {:transform_id tb-id
+ :dag_run_id dag-run-id
+ :status "failed"
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-02T10:00:00")}
+ :model/TransformRun {standalone-id :id} {:transform_id ta-id
+ :status "succeeded"
+ :run_method "manual"
+ :user_id (mt/user->id :lucky)
+ :start_time (parse-instant "2025-09-03T10:00:00")
+ :end_time (parse-instant "2025-09-03T10:01:00")}]
+ (f {:ta-id ta-id, :tb-id tb-id, :job-id job-id, :job-run-id job-run-id, :dag-run-id dag-run-id
+ :member-a-id member-a-id, :member-b-id member-b-id, :standalone-id standalone-id})))
+
+(defmacro ^:private with-runs-fixture!
+ "Execute `body` with `binding` bound to the fixture-id map of [[do-with-runs-fixture!]]."
+ [[binding] & body]
+ `(do-with-runs-fixture! (fn [~binding] ~@body)))
+
+(defn- get-runs [& params]
+ ;; a large limit so fixture rows can't be pushed off the page by rows other tests created
+ (apply mt/user-http-request :lucky :get 200 "transform/runs" :limit 1000 params))
+
+(deftest unified-runs-listing-test
+ (testing "GET /api/transform/runs"
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (with-runs-fixture! [{:keys [ta-id tb-id job-id job-run-id dag-run-id member-a-id member-b-id standalone-id]}]
+ (let [response (get-runs)
+ rows (rows-by-type response)]
+ (testing "returns all three kinds of root run"
+ (is (contains? rows ["job" job-run-id]))
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (contains? rows ["transform" standalone-id])))
+ (testing "member runs of a job/DAG run are not listed as rows"
+ (is (not (contains? rows ["transform" member-a-id])))
+ (is (not (contains? rows ["transform" member-b-id]))))
+ (testing "pagination envelope"
+ (is (<= 3 (:total response)))
+ (is (pos? (:limit response)))
+ (is (integer? (:offset response))))
+ (testing "job row shape"
+ (let [row (rows ["job" job-run-id])]
+ (is (= job-id (:entity_id row)))
+ (is (= "Nightly Job" (:name row)))
+ (is (nil? (:direction row)))
+ (is (nil? (:transform_count row)))
+ (is (= "cron" (:run_method row)))
+ (is (= "succeeded" (:status row)))
+ (is (= (utc-timestamp "2025-09-01T10:00:00") (:start_time row)))
+ (is (= (utc-timestamp "2025-09-01T10:05:00") (:end_time row)))))
+ (testing "dag row shape"
+ (let [row (rows ["dag" dag-run-id])]
+ (is (= tb-id (:entity_id row)))
+ (is (= "Transform B" (:name row)))
+ (is (= "downstream" (:direction row)))
+ (is (= 4 (:transform_count row)))
+ (is (= "manual" (:run_method row)))
+ (is (= "failed" (:status row)))
+ (is (= "boom" (:message row)))
+ (is (= (mt/user->id :lucky) (:user_id row)))))
+ (testing "standalone transform row shape"
+ (let [row (rows ["transform" standalone-id])]
+ (is (= ta-id (:entity_id row)))
+ (is (= "Transform A" (:name row)))
+ (is (nil? (:direction row)))
+ (is (= "manual" (:run_method row)))
+ (is (= (mt/user->id :lucky) (:user_id row)))))))))))
+
+(deftest unified-runs-filters-test
+ (testing "GET /api/transform/runs filters"
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (with-runs-fixture! [{:keys [ta-id tb-id job-run-id dag-run-id standalone-id]}]
+ (testing "types= restricts the kinds returned"
+ (let [response (get-runs :types ["job"])]
+ (is (every? #(= "job" (:run_type %)) (:data response)))
+ (is (contains? (rows-by-type response) ["job" job-run-id])))
+ (let [response (get-runs :types ["dag" "transform"])
+ rows (rows-by-type response)]
+ (is (every? #(contains? #{"dag" "transform"} (:run_type %)) (:data response)))
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (contains? rows ["transform" standalone-id]))))
+ (testing "statuses= filters across all kinds, matching any of the given statuses"
+ (let [rows (rows-by-type (get-runs :statuses ["failed"]))]
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (not (contains? rows ["job" job-run-id])))
+ (is (not (contains? rows ["transform" standalone-id]))))
+ (let [rows (rows-by-type (get-runs :statuses ["failed" "succeeded"]))]
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (contains? rows ["job" job-run-id]))
+ (is (contains? rows ["transform" standalone-id]))))
+ (testing "run-methods= filters by trigger"
+ (let [rows (rows-by-type (get-runs :run-methods ["cron"]))]
+ (is (contains? rows ["job" job-run-id]))
+ (is (not (contains? rows ["dag" dag-run-id])))
+ (is (not (contains? rows ["transform" standalone-id]))))
+ (let [rows (rows-by-type (get-runs :run-methods ["manual"]))]
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (contains? rows ["transform" standalone-id]))
+ (is (not (contains? rows ["job" job-run-id])))))
+ (testing "transform-ids= returns runs that ran any of the transforms"
+ (testing "job runs with a member run of it, plus its standalone runs"
+ (let [rows (rows-by-type (get-runs :transform-ids [ta-id]))]
+ (is (contains? rows ["job" job-run-id]))
+ (is (contains? rows ["transform" standalone-id]))
+ (is (not (contains? rows ["dag" dag-run-id])))))
+ (testing "DAG runs with a member run of it"
+ (let [rows (rows-by-type (get-runs :transform-ids [tb-id]))]
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (not (contains? rows ["job" job-run-id])))
+ (is (not (contains? rows ["transform" standalone-id])))))
+ (testing "multiple ids match as a logical OR"
+ (let [rows (rows-by-type (get-runs :transform-ids [ta-id tb-id]))]
+ (is (contains? rows ["job" job-run-id]))
+ (is (contains? rows ["dag" dag-run-id]))
+ (is (contains? rows ["transform" standalone-id])))))
+ (testing "start-time= filters by run start"
+ (let [rows (rows-by-type (get-runs :start-time "2025-09-03"))]
+ (is (contains? rows ["transform" standalone-id]))
+ (is (not (contains? rows ["job" job-run-id])))
+ (is (not (contains? rows ["dag" dag-run-id])))))
+ (testing "end-time= filters by run end"
+ (let [rows (rows-by-type (get-runs :end-time "2025-09-01"))]
+ (is (contains? rows ["job" job-run-id]))
+ (is (not (contains? rows ["dag" dag-run-id])))
+ (is (not (contains? rows ["transform" standalone-id]))))))))))
+
+(deftest unified-runs-sorting-and-pagination-test
+ (testing "GET /api/transform/runs sorting and pagination"
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (with-runs-fixture! [{:keys [job-run-id dag-run-id standalone-id]}]
+ (testing "sort-column=start_time asc keeps our rows in chronological order"
+ (let [response (get-runs :sort-column "start_time" :sort-direction "asc")
+ ours (filter (comp #{["job" job-run-id] ["dag" dag-run-id] ["transform" standalone-id]}
+ (juxt :run_type :id))
+ (:data response))]
+ ;; rows from other tests may interleave; the relative order of ours must hold
+ (is (= [["job" job-run-id] ["dag" dag-run-id] ["transform" standalone-id]]
+ (map (juxt :run_type :id) ours)))))
+ (testing "limit/offset are honored"
+ (let [response (mt/user-http-request :lucky :get 200 "transform/runs" :limit 1 :offset 0)]
+ (is (= 1 (count (:data response))))
+ (is (= 1 (:limit response)))
+ (is (= 0 (:offset response)))
+ (is (<= 3 (:total response))))))))))
+
+(deftest unified-runs-deleted-entities-test
+ (testing "GET /api/transform/runs keeps rows for deleted jobs/transforms, naming them from the run-start snapshot"
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Transform {ta-id :id} {:name "Doomed Transform"}
+ :model/Transform {tb-id :id} {:name "Doomed Seed"}
+ :model/TransformJob {job-id :id} {:name "Doomed Job" :schedule "0 0 0 * * ?"}
+ :model/TransformJobRun {job-run-id :id} {:job_id job-id
+ :job_name "Doomed Job"
+ :status "succeeded"
+ :run_method "cron"
+ :start_time (parse-instant "2025-09-01T10:00:00")}
+ :model/TransformDagRun {dag-run-id :id} {:source_transform_id tb-id
+ :source_transform_name "Doomed Seed"
+ :direction "upstream"
+ :status "succeeded"
+ :start_time (parse-instant "2025-09-02T10:00:00")}
+ :model/TransformRun {standalone-id :id} {:transform_id ta-id
+ :transform_name "Doomed Transform"
+ :status "succeeded"
+ :run_method "manual"
+ :start_time (parse-instant "2025-09-03T10:00:00")}]
+ (t2/delete! :model/TransformJob :id job-id)
+ (t2/delete! :model/Transform :id [:in [ta-id tb-id]])
+ (let [rows (rows-by-type (get-runs))]
+ (testing "job run survives (job_id has no FK) with its snapshot name"
+ (let [row (rows ["job" job-run-id])]
+ (is (some? row))
+ (is (= "Doomed Job" (:name row)))))
+ (testing "DAG run survives (FK is SET NULL) with a nil entity_id and its snapshot name"
+ (let [row (rows ["dag" dag-run-id])]
+ (is (some? row))
+ (is (nil? (:entity_id row)))
+ (is (= "Doomed Seed" (:name row)))))
+ (testing "standalone run survives (FK is SET NULL) with a nil entity_id and its snapshot name"
+ (let [row (rows ["transform" standalone-id])]
+ (is (some? row))
+ (is (nil? (:entity_id row)))
+ (is (= "Doomed Transform" (:name row)))))))))))
+
+(deftest unified-runs-requires-data-analyst-test
+ (testing "GET /api/transform/runs requires the data-analyst role"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/user-http-request :rasta :get 403 "transform/runs"))))