Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add schemas to transform and transform-test modules (#86196)

Alexander Polyankin18656affb45b
diff --git a/.claude/skills/_shared/clojure-style-guide.md b/.claude/skills/_shared/clojure-style-guide.md
index 60d58e11e99..d0c8f0b6993 100644
--- a/.claude/skills/_shared/clojure-style-guide.md
+++ b/.claude/skills/_shared/clojure-style-guide.md
@@ -212,6 +212,25 @@ destructuring in particular, since it names the local after the key. kondo's `:s
- Mark pure function tests `^:parallel`
+**Build queries with Lib:**
+
+Every query in a test, fixture data included (`mt/with-temp` sources, `:dataset_query`, request bodies), is built with
+`metabase.lib`: `lib/query`, `lib/native-query`, `lib/with-template-tags`, and columns from the metadata provider. Never
+a hand-written map, neither legacy (`{:database … :type :native …}`, `:source-table`) nor MBQL 5
+(`{:lib/type :mbql/query :stages […]}`).
+
+```clojure
+;; bad
+{:lib/type :mbql/query :database (mt/id) :stages [{:lib/type :mbql.stage/native :native "SELECT 1"}]}
+{:database (mt/id) :type :native :native {:query "SELECT 1"}}
+
+;; good
+(lib/native-query (mt/metadata-provider) "SELECT 1")
+(lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")
+(let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id :orders))))
+```
+
**Redefining `defn`-vars under test:**
Use `mt/with-dynamic-fn-redefs` instead of `with-redefs` when stubbing `defn`/`defn-` vars — plain `with-redefs` is not parallel-test-safe and a clj-kondo hook (`:metabase/prefer-with-dynamic-fn-redefs`, see `.clj-kondo/config.edn`) will warn. `with-redefs` is still fine for `def` / dynamic var rebinding.
diff --git a/.claude/skills/clojure-review/SKILL.md b/.claude/skills/clojure-review/SKILL.md
index 4bc2bb147e5..0fc8d300f26 100644
--- a/.claude/skills/clojure-review/SKILL.md
+++ b/.claude/skills/clojure-review/SKILL.md
@@ -82,6 +82,7 @@ liability it usually is.
- [ ] Separate `deftest` forms for distinct test cases
- [ ] Pure tests marked `^:parallel`
- [ ] Test names end in `-test` or `-test-<number>`
+- [ ] Test queries, fixtures included, built with `lib/` (`lib/query`, `lib/native-query`), never hand-written maps
### Modules
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index aa9841190a4..94385991fdc 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3111,7 +3111,7 @@
transforms.feature-gating
{:team "Graphy"
:api #{metabase.transforms.feature-gating}
- :uses #{api premium-features}}
+ :uses #{api premium-features util}}
transforms.indexes
{:team "Gadget"
@@ -3349,6 +3349,7 @@
:model/ImplicitAction
:model/QueryAction
:model/Table
+ :model/Transform
:model/User}}
warehouses.rest
@@ -3952,7 +3953,6 @@
:model/DashboardCardSeries
:model/DashboardTab
:model/DataApp
- :model/Database
:model/Dimension
:model/Document
:model/Field
@@ -4581,16 +4581,21 @@
driver
events
lib
+ lib.be
lib.schema
models
+ permissions
premium-features
+ query-permissions
query-processor
+ remote-sync
request
revisions
run-tracking
sql-tools
sql-tools.parsing
task
+ transforms
transforms.base
util}
:model-exports #{:model/TransformTest}
diff --git a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
index 078b211c8a6..a2fb3786f5a 100644
--- a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
+++ b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
@@ -180,7 +180,7 @@ describe("admin > database > database routing", () => {
cy.findByTestId("database-name-confirmation-input").type(
"Destination DB 1 Updated",
);
- cy.button("Delete this DB connection").click();
+ cy.button("Delete").click();
cy.wait("@deleteDatabase");
});
dbRoutingSection()
diff --git a/e2e/test/scenarios/admin/databases.cy.spec.js b/e2e/test/scenarios/admin/databases.cy.spec.js
index 0b8be492789..050a4852d87 100644
--- a/e2e/test/scenarios/admin/databases.cy.spec.js
+++ b/e2e/test/scenarios/admin/databases.cy.spec.js
@@ -488,7 +488,7 @@ describe("scenarios > admin > databases > sample database", () => {
cy.findByTestId("database-name-confirmation-input").type(
"Sample Database",
);
- cy.findByText("Delete this DB connection").click();
+ cy.findByText("Delete this content and the DB connection").click();
cy.wait("@deleteDatabase");
});
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index 3ade540f927..2a41dfa0a8d 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -1382,66 +1382,6 @@ LIMIT
});
});
- describe("disconnected database", () => {
- it("should warn about transforms when deleting a database and show disconnected banner on transform pages", () => {
- cy.log("create a transform");
- createMbqlTransform({ visitTransform: false });
-
- cy.log("go to admin and delete the writable database");
- cy.intercept("GET", "/api/database/*/usage_info").as("usageInfo");
- cy.intercept("DELETE", "/api/database/*").as("deleteDb");
- cy.visit(`/admin/databases/${WRITABLE_DB_ID}`);
- cy.button("Remove this database").click();
- cy.wait("@usageInfo");
-
- cy.log(
- "verify the delete modal warns about transforms that will stop working",
- );
- H.modal().within(() => {
- cy.findByLabelText(/1 transform will stop working/)
- .should("not.be.checked")
- .click()
- .should("be.checked");
- cy.findByTestId("database-name-confirmation-input").type(DB_NAME);
- cy.findByText("Delete this DB connection").click();
- cy.wait("@deleteDb");
- });
-
- cy.log(
- "visit the transform query page and verify the disconnected banner",
- );
- cy.visit("/data-studio/transforms/1");
- verifyDisconnectedDatabaseBanner();
-
- cy.log("edit definition button should not be visible");
- H.DataStudio.Transforms.editDefinitionButton().should("not.exist");
-
- cy.log(
- "visit the run page and verify the disconnected banner is visible",
- );
- H.DataStudio.Transforms.runTab().click();
- verifyDisconnectedDatabaseBanner();
-
- cy.log(
- "visit the settings page and verify the disconnected banner is visible",
- );
- H.DataStudio.Transforms.settingsTab().click();
- verifyDisconnectedDatabaseBanner();
-
- cy.log(
- "visit the inspect page and verify the disconnected banner is visible",
- );
- H.DataStudio.Transforms.inspectTab().click();
- verifyDisconnectedDatabaseBanner();
-
- cy.log(
- "visit the dependencies page and verify the disconnected banner is visible",
- );
- H.DataStudio.Transforms.dependenciesTab().click();
- verifyDisconnectedDatabaseBanner();
- });
- });
-
describe("cancelation", () => {
function createSlowTransform(seconds: number = 100) {
H.createTransform(
@@ -3313,16 +3253,6 @@ describe("scenarios > admin > transforms", () => {
});
});
-function verifyDisconnectedDatabaseBanner() {
- return cy
- .findByRole("alert")
- .should("be.visible")
- .and(
- "contain.text",
- "The database this transform depends on has been disconnected",
- );
-}
-
function getTransformsNavLink() {
return H.DataStudio.nav().findByRole("link", { name: "Data transformation" });
}
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj
index c7a44f37411..32081621aaa 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis_finding.clj
@@ -35,20 +35,6 @@
[entity]
(some-> entity :definition lib/database-id))
-(defmulti ^:private pre-analysis-errors
- "The set of errors of `entity` found without running the analysis, such as an orphan Transform's, which when not
- empty are recorded instead of running it."
- {:arglists '([entity])}
- t2/model)
-
-(defmethod pre-analysis-errors :default [_] nil)
-
-(defmethod pre-analysis-errors :model/Transform
- [{db-id :source_database_id id :id}]
- (when (nil? db-id)
- #{(-> (lib/validation-exception-error "Source database for this transform has been deleted.")
- (assoc :source-entity-type :transform :source-entity-id id))}))
-
(mu/defn upsert-analysis!
"Analyze the Toucan entity `entity` and insert or replace its AnalysisFinding, reusing the caller's metadata provider
cache when it has one."
@@ -57,19 +43,17 @@
(let [model (t2/model entity)
entity-type (deps.dependency-types/model->dependency-type model)
entity-id (:id entity)]
- (if-let [errors (not-empty (pre-analysis-errors entity))]
- (models.analysis-finding/upsert-analysis! entity-type entity-id false errors)
- (if-let [db-id (entity-db-id entity)]
- (let [mp (lib-be/application-database-metadata-provider db-id)
- results (try (deps.analysis/check-entity mp entity-type entity-id)
- (catch Exception e
- (log/errorf "Error analyzing entity: %s" (ex-message e))
- #{(lib/validation-exception-error (.getMessage e))}))
- success (empty? results)]
- (models.analysis-finding/upsert-analysis! entity-type entity-id success results))
- (models.analysis-finding/upsert-analysis!
- entity-type entity-id false
- #{(lib/validation-exception-error "Could not resolve a database for this entity.")}))))))
+ (if-let [db-id (entity-db-id entity)]
+ (let [mp (lib-be/application-database-metadata-provider db-id)
+ results (try (deps.analysis/check-entity mp entity-type entity-id)
+ (catch Exception e
+ (log/errorf "Error analyzing entity: %s" (ex-message e))
+ #{(lib/validation-exception-error (.getMessage e))}))
+ success (empty? results)]
+ (models.analysis-finding/upsert-analysis! entity-type entity-id success results))
+ (models.analysis-finding/upsert-analysis!
+ entity-type entity-id false
+ #{(lib/validation-exception-error "Could not resolve a database for this entity.")})))))
(mu/defn- analyze-entity!
"Analyze `entity`, recording a failure as a failed finding before rethrowing it."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index d88966d0d6c..2a9a88bb87c 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -695,11 +695,6 @@
[field-id :- ::lib.schema.id/field]
(t2/select-one-fn :table_id :model/Field :id field-id))
-(mu/defn source-database-transform-ids :- [:set ::lib.schema.id/transform]
- "The IDs of the Transforms reading from the Database with `db-id`."
- [db-id :- ::lib.schema.id/database]
- (or (t2/select-pks-set :model/Transform :source_database_id db-id) #{}))
-
;;; -------------------------------------------------- Dependencies --------------------------------------------------
(mu/defn dependencies-from
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
index 3b53e711d6d..1c4959bcf79 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
@@ -312,8 +312,8 @@
(trigger-jobs!)))
(defenterprise before-delete
- "Mark what deleting `row` of `model` changes: the entity it is a part of, what reads a deleted entity, or
- the transforms reading a deleted Database; and trigger the jobs once it commits."
+ "Mark what deleting `row` of `model` changes, the entity it is a part of and what reads a deleted entity, and
+ trigger the jobs once it commits."
:feature :dependencies
[model row]
(when (contains? part-columns model)
@@ -327,10 +327,6 @@
nil)
(when-let [entity-type (deps.dependency-types/model->dependency-type model)]
(mark-output-changed! entity-type row))
- (when (= model :model/Database)
- (when-let [transform-ids (not-empty (dependencies.db/source-database-transform-ids (:id row)))]
- (dependencies.db/mark-dependency-statuses-stale! {:transform transform-ids})
- (dependencies.db/mark-analysis-finding-statuses-stale! {:transform transform-ids})))
(trigger-jobs!))
(defenterprise after-transform-run
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/api.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/api.clj
index 73c9d38f5a5..c504df093d7 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/api.clj
@@ -10,20 +10,22 @@
[metabase.models.interface :as mi]
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.request.core :as request]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn- refusing
+(mu/defn- refusing :- [:maybe ::transform-testing.schema/run-result]
"Call `thunk`, rethrowing a typed transform-test refusal as the status and `:error-code` it publishes.
400 — the test is wrong, and its author can fix it.
422 — the test is fine; the transform or its database prevents a run here.
501 — the test asks for something not built yet."
- [thunk]
+ [thunk :- ifn?]
(try
(thunk)
(catch clojure.lang.ExceptionInfo e
@@ -47,7 +49,7 @@
"Get a transform test with its newest run."
[{:keys [id]} :- [:map {:closed true}
[:id ms/PositiveInt]]]
- (-> (api/read-check (transform-testing.db/transform-test id))
+ (-> (api/read-check :model/TransformTest id)
(t2/hydrate :last_run)))
(api.macros/defendpoint :post "/" :- ::transform-testing.schema/transform-test
@@ -55,8 +57,10 @@
[_route-params
_query-params
{:keys [transform_id inputs expectations] :as body} :- ::transform-testing.schema/transform-test.create]
- (api/write-check :model/Transform transform_id)
- (refusing #(transform-testing.runner/validate-transform-test transform_id inputs expectations))
+ (api/create-check :model/TransformTest body)
+ (refusing #(transform-testing.runner/validate-test (transforms.u/->definition (transform-testing.db/transform transform_id))
+ inputs
+ expectations))
(transform-testing.db/insert-transform-test! (assoc body :creator_id api/*current-user-id*)))
(api.macros/defendpoint :put "/:id" :- ::transform-testing.schema/transform-test
@@ -64,14 +68,15 @@
[{:keys [id]} :- [:map {:closed true}
[:id ms/PositiveInt]]
_query-params
- {:keys [transform_id] :as body} :- ::transform-testing.schema/transform-test.update]
- (let [transform-test (api/write-check (transform-testing.db/transform-test id))]
- (when transform_id
- (api/write-check :model/Transform transform_id))
+ body :- ::transform-testing.schema/transform-test.update]
+ (let [transform-test (api/read-check :model/TransformTest id)]
+ (api/update-check transform-test body)
+ (when (some #(contains? body %) [:transform_id :inputs :expectations])
+ (let [{:keys [transform_id inputs expectations]} (merge transform-test body)]
+ (refusing #(transform-testing.runner/validate-test (transforms.u/->definition (transform-testing.db/transform transform_id))
+ inputs
+ expectations))))
(when (seq body)
- (when (some #(contains? body %) [:transform_id :inputs :expectations])
- (let [{:keys [transform_id inputs expectations]} (merge transform-test body)]
- (refusing #(transform-testing.runner/validate-transform-test transform_id inputs expectations))))
(transform-testing.db/update-transform-test! id body)))
(transform-testing.db/transform-test id))
@@ -79,13 +84,13 @@
"Delete a transform test."
[{:keys [id]} :- [:map {:closed true}
[:id ms/PositiveInt]]]
- (api/write-check (transform-testing.db/transform-test id))
+ (api/write-check :model/TransformTest id)
(transform-testing.db/delete-transform-test! id)
nil)
-(defn- timestamp-range
+(mu/defn- timestamp-range :- [:maybe [:tuple [:maybe ms/TemporalInstant] [:maybe ms/TemporalInstant]]]
"The `[start end]` instants of the date filter `date-string`, or nil when it is nil."
- [date-string]
+ [date-string :- [:maybe :string]]
(when date-string
(let [{:keys [start end]} (try
(params.dates/date-string->range date-string {:inclusive-end? false})
@@ -103,7 +108,7 @@
[:statuses {:optional true} [:maybe (ms/QueryVectorOf ::transform-testing.schema/run-status)]]
[:start-time {:optional true} [:maybe ms/NonBlankString]]
[:end-time {:optional true} [:maybe ms/NonBlankString]]]]
- (api/read-check (transform-testing.db/transform-test id))
+ (api/read-check :model/TransformTest id)
(let [[started-at-start started-at-end] (timestamp-range start-time)
[ended-at-start ended-at-end] (timestamp-range end-time)
filters {:transform-test-id id
@@ -120,7 +125,7 @@
:total (transform-testing.db/transform-test-run-count filters)}))
(api.macros/defendpoint :post "/:id/run" :- ::transform-testing.schema/run-result
- "Run a transform test.
+ "Run a transform test, which reading it is enough to do.
A 200 means the run happened: `:status` is then `passed` or `failed`, and every expectation
reports what it found — including one that could not be evaluated, which comes back as that
@@ -130,9 +135,9 @@
naming which refusal it was, as [[refusing]] describes."
[{:keys [id]} :- [:map {:closed true}
[:id ms/PositiveInt]]]
- (refusing #(-> (transform-testing.db/transform-test id)
- api/write-check
- transform-testing.runner/run-transform-test!)))
+ (let [transform-test (api/read-check :model/TransformTest id)
+ transform (api/read-check :model/Transform (:transform_id transform-test))]
+ (refusing #(transform-testing.runner/run-saved-test! transform transform-test))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/transform-test` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/compile.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/compile.clj
index 0eb3afc87f7..44f2beb63b7 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/compile.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/compile.clj
@@ -130,7 +130,8 @@
(defn- compiled
"`honeysql` formatted for `driver` as a compiled query."
- [driver honeysql]
+ [driver
+ honeysql]
(let [[query & params] (sql.qp/format-honeysql driver honeysql)]
{:query query :params (vec params)}))
@@ -148,7 +149,8 @@
"`alias-name` as a column or table alias, also the shape a bare expression takes as a `:select` entry."
([alias-name]
[alias-name])
- ([identifier-type alias-name]
+ ([identifier-type
+ alias-name]
[(h2x/identifier identifier-type alias-name)]))
(def ^:private no-rows
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
index 7add856c762..6cb18c729bf 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
@@ -46,8 +46,8 @@
[:ended-at-start {:optional true} [:maybe ms/TemporalInstant]]
[:ended-at-end {:optional true} [:maybe ms/TemporalInstant]]])
-(defn- run-filter-clause
- [{:keys [transform-test-id statuses started-at-start started-at-end ended-at-start ended-at-end]}]
+(mu/defn- run-filter-clause :- vector?
+ [{:keys [transform-test-id statuses started-at-start started-at-end ended-at-start ended-at-end]} :- RunFilters]
(cond-> [:and [:= :transform_test_id transform-test-id]]
(seq statuses) (conj [:in :status (map name statuses)])
started-at-start (conj [:>= :start_time started-at-start])
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/errors.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/errors.clj
index 760374e630b..05e850483ba 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/errors.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/errors.clj
@@ -5,7 +5,9 @@
A refusal and a failure are different outcomes, and a caller that confuses them either gives up
on a real bug or chases one that does not exist. Everything here is a refusal: the run did not
happen, or could not be completed. A failing expectation is not an error — it is a result, and
- it rides back on the expectation that produced it.")
+ it rides back on the expectation that produced it."
+ (:require
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -38,9 +40,9 @@
"Every declared `:error-type`."
(set (keys error-types)))
-(defn checked
+(mu/defn checked :- :keyword
"Return `error-type` iff it is a member of [[all]]; otherwise throw."
- [error-type]
+ [error-type :- :keyword]
(when-not (contains? all error-type)
(throw (ex-info (str error-type " is not a declared transform-test error type; see"
" metabase-enterprise.transform-testing.errors/all.")
@@ -59,25 +61,26 @@
`(ex-info ~msg (assoc ~data :error-type ~error-type) ~cause))
`(ex-info ~msg (assoc ~data :error-type (checked ~error-type)) ~cause))))
-(defn code
+(mu/defn code :- :string
"The public wire spelling of `error-type` — the vocabulary a client switches on.
Derived from the keyword's name so the two cannot drift, and prefixed so that the Clojure
namespace this vocabulary happens to live in is not what gets published."
- [error-type]
+ [error-type :- :keyword]
(str "transform-test." (name error-type)))
-(defn status-code
+(mu/defn status-code :- :int
"The HTTP status for `error-type`, or 500 for an untyped or unknown one."
- [error-type]
+ [error-type :- [:maybe :keyword]]
(get error-types error-type 500))
-(defn remap-message
+(mu/defn remap-message :- [:maybe :string]
"`message` with every temp-table name in `temp->logical` replaced by the name the author wrote.
Matching ignores case because Snowflake and H2 upper-case the generated names. A nil message
stays nil."
- [message temp->logical]
+ [message :- [:maybe :string]
+ temp->logical :- [:map-of :string :string]]
(when message
(reduce (fn [^String msg [temp logical]]
(.replaceAll msg (str "(?i)\\b" (java.util.regex.Pattern/quote temp) "\\b")
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/equals.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/equals.clj
index 0b22c5ef039..b7975c5b9e7 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/equals.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/equals.clj
@@ -14,7 +14,8 @@
[metabase-enterprise.transform-testing.schema :as transform-testing.schema]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
@@ -24,6 +25,17 @@
"A value as a failure report carries it: a string, a number, a boolean, or null."
[:maybe [:or :boolean number? :string]])
+(mr/def ::cell-mismatch
+ "A cell that differs between the one missing row and the one extra row."
+ [:map {:closed true}
+ [:column :string]
+ [:expected cell]
+ [:actual cell]])
+
+(mr/def ::expectation
+ "A built `equals` expectation."
+ [:fn #(satisfies? expectations.protocol/Expectation %)])
+
(def ^:private base
"The shared result keys, pinned to this type."
[:merge
@@ -40,10 +52,7 @@
[:expected :int]]]
[:extra-rows [:sequential ::transform-testing.schema/row]]
[:missing-rows [:sequential ::transform-testing.schema/row]]
- [:cell-mismatches [:sequential [:map {:closed true}
- [:column :string]
- [:expected cell]
- [:actual cell]]]]
+ [:cell-mismatches [:sequential ::cell-mismatch]]
[:truncated :int]]])
(mr/def ::result
@@ -61,13 +70,15 @@
;;; --------------------------------------------- Reporting ---------------------------------------------
-(defn- cell-mismatches
+(mu/defn- cell-mismatches :- [:maybe [:sequential ::cell-mismatch]]
"The cells that differ between the one missing row and the one extra row, or nil when the diff is
not one-for-one.
Only the unambiguous case is paired. Sorting both sides and pairing positionally for a larger
diff produces output that reads as authoritative and is arbitrary."
- [column-names [missing :as missing-rows] [extra :as extra-rows]]
+ [column-names :- [:sequential :string]
+ [missing :as missing-rows] :- [:sequential ::transform-testing.schema/row]
+ [extra :as extra-rows] :- [:sequential ::transform-testing.schema/row]]
(when (= 1 (count missing-rows) (count extra-rows))
(not-empty
(vec (for [c column-names
@@ -77,7 +88,7 @@
;;; ---------------------------------------- Resolving declared columns ----------------------------------
-(defn- resolve-column
+(mu/defn- resolve-column :- :string
"The output table's own spelling of `declared-name`.
The author writes the name as it appears in the transform's SQL; the warehouse stores whatever its
@@ -85,7 +96,9 @@
and lower on Postgres. Resolving against the materialized table means the comparison names a
column the table actually has — and, since the result is always one of the output's own names, no
author-supplied string ever reaches the SQL as an identifier."
- [expectation-name output-columns declared-name]
+ [expectation-name :- :string
+ output-columns :- [:sequential :string]
+ declared-name :- :string]
(let [folded (filter #(.equalsIgnoreCase ^String % declared-name) output-columns)]
(or (first (filter #(= % declared-name) output-columns))
(when (= 1 (count folded)) (first folded))
@@ -110,12 +123,13 @@
:column declared-name
:available (vec output-columns)}))))))
-(defn resolve-columns
+(mu/defn resolve-columns :- [:sequential :string]
"The output table's own spelling of each column `expectation` declares, in declared order.
Only the SQL uses these. A failure reports the column under the name the author wrote, which is
what they can find in their own test."
- [{:keys [name columns]} output-columns]
+ [{:keys [name columns]} :- ::expectation
+ output-columns :- [:sequential ::transform-testing.schema/result-column]]
(let [output-names (mapv :name output-columns)]
(mapv #(resolve-column name output-names (:name %)) columns)))
@@ -175,7 +189,8 @@
(cell-mismatches column-names missing extra)
(assoc :cell-mismatches (cell-mismatches column-names missing extra))))))
-(defn- not-implemented [expectation-name]
+(mu/defn- not-implemented :- (ms/InstanceOfClass clojure.lang.ExceptionInfo)
+ [expectation-name :- :string]
(transform-testing.errors/ex
::transform-testing.errors/unsupported-format
(tru "Expectation {0} compares against a SQL query, which is not implemented yet. Give ''columns'' and ''rows'' instead."
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/report.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/report.clj
index 8ff5f871e19..0e2a9c662d3 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/report.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/expectations/report.clj
@@ -2,6 +2,9 @@
"Turning warehouse values into something a failure report can carry.
Shared by every expectation type, so it sits below them rather than in the front door."
+ (:require
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms])
(:import
(java.math BigDecimal)))
@@ -11,12 +14,12 @@
"How many rows of any one failure list are reported. The rest are counted as truncated."
50)
-(defn cell
+(mu/defn cell :- [:maybe [:or :boolean number? :string]]
"`v` as something JSON can carry.
A BigDecimal keeps its scale: when scale is the difference between expected and actual,
normalizing it away makes the report deny the very thing it is reporting."
- [v]
+ [v :- ms/FieldValue]
(cond
(nil? v) nil
(instance? BigDecimal v) (.toPlainString ^BigDecimal v)
@@ -25,7 +28,7 @@
(string? v) v
:else (str v)))
-(defn capped
+(mu/defn capped :- [:tuple [:vector [:sequential ms/FieldValue]] :int]
"`[first-n dropped]` — `rows` cut to [[row-cap]], and how many that lost."
- [rows]
+ [rows :- [:sequential [:sequential ms/FieldValue]]]
[(vec (take row-cap rows)) (max 0 (- (count rows) row-cap))])
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
index cd0f1fd9f79..748b830d3fc 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
@@ -2,13 +2,22 @@
(:require
[metabase-enterprise.transform-testing.db :as transform-testing.db]
[metabase-enterprise.transform-testing.schema :as transform-testing.schema]
+ [metabase.api.common :as api]
[metabase.audit-app.core :as audit-app]
[metabase.events.core :as events]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
+ [metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.query-permissions.core :as query-perms]
+ [metabase.remote-sync.core :as remote-sync]
[metabase.revisions.models.revision :as revision]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.malli :as mu]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -20,10 +29,10 @@
(derive :hook/entity-id)
(derive :hook/timestamped?))
-(defn- json-column
+(mu/defn- json-column :- [:map {:closed true} [:in fn?] [:out fn?]]
"A Toucan transform storing a value of `schema` as JSON, normalized and validated on its way in and normalized on its
way out."
- [schema]
+ [schema :- :keyword]
{:in (comp mi/json-in #(mu/validate-throw schema %) #(lib/normalize schema %))
:out (comp #(lib/normalize schema %) mi/json-out-with-keywordization)})
@@ -52,6 +61,30 @@
;;; ------------------------------------------------- Permissions --------------------------------------------------
+(mu/defn can-run-test? :- :boolean
+ "Whether the current user may run test SQL they write against the source Database of `transform`, as an ad hoc
+ native query and a transform there need."
+ [transform :- ::transforms.schema/transform.definition]
+ (let [db-id (transforms-base.i/source-db-id transform)]
+ (boolean
+ (or api/*is-superuser?*
+ (and db-id
+ (transforms.u/source-readable? transform)
+ (perms/has-db-transforms-permission? api/*current-user-id* db-id)
+ (query-perms/can-run-query?
+ (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")))))))
+
+(mu/defn- can-save-test? :- :boolean
+ "Whether the current user may save or delete a test of the Transform `transform-id`, while remote sync allows editing
+ transforms."
+ [transform-id :- ::lib.schema.id/transform]
+ (boolean
+ (and (remote-sync/transforms-editable?)
+ (when-let [transform (transform-testing.db/transform transform-id)]
+ (or api/*is-superuser?*
+ (and (transforms-base.i/source-db-id transform)
+ (can-run-test? (transforms.u/->definition transform))))))))
+
(defmethod mi/can-read? :model/TransformTest
([instance]
(mi/can-read? :model/Transform (:transform_id instance)))
@@ -61,14 +94,21 @@
(defmethod mi/can-write? :model/TransformTest
([instance]
- (mi/can-write? :model/Transform (:transform_id instance)))
+ (can-save-test? (:transform_id instance)))
([_model pk]
(when-let [transform-test (transform-testing.db/transform-test pk)]
(mi/can-write? transform-test))))
(defmethod mi/can-create? :model/TransformTest
[_model instance]
- (mi/can-write? :model/Transform (:transform_id instance)))
+ (can-save-test? (:transform_id instance)))
+
+(defmethod mi/can-update? :model/TransformTest
+ [instance changes]
+ (let [transform-id (:transform_id changes (:transform_id instance))]
+ (and (can-save-test? (:transform_id instance))
+ (or (= transform-id (:transform_id instance))
+ (can-save-test? transform-id)))))
;;; -------------------------------------------------- Revisions ---------------------------------------------------
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj
index 32126c98de4..e3942444d55 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj
@@ -43,12 +43,12 @@
(finally
(swap! active-runs disj run-id))))
-(defn heartbeat-runs!
+(mu/defn heartbeat-runs!
"Stamp a heartbeat on the given runs while they are still started."
- [run-ids]
+ [run-ids :- [:sequential ms/PositiveInt]]
(rt/heartbeat-ids! :model/TransformTestRun [:status "started"] :last_heartbeat run-ids))
-(defn heartbeat-and-reconcile-runs!
+(mu/defn heartbeat-and-reconcile-runs! :- :nil
"Heartbeat the runs owned by this process and forget any that became terminal elsewhere."
[]
(rt/heartbeat-and-reconcile! {:model :model/TransformTestRun
@@ -57,9 +57,9 @@
:heartbeat! heartbeat-runs!
:on-gone #(swap! active-runs disj %)}))
-(defn reap-orphaned-runs!
+(mu/defn reap-orphaned-runs! :- [:maybe [:sequential ::transform-testing.schema/transform-test-run]]
"Time out started runs whose owning process has stopped heartbeating."
- [stale-minutes]
+ [stale-minutes :- ms/PositiveInt]
(let [reaped (rt/reap-orphaned! {:model :model/TransformTestRun
:active [:status "started"]
:stale [{:column :last_heartbeat :age stale-minutes :unit :minute}]
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
index 64da975f4d9..fb88d4e67b8 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
@@ -1,9 +1,9 @@
(ns metabase-enterprise.transform-testing.runner
- "Orchestrates a transform test run over the module's separated concerns. The runner is the only
- namespace that resolves from the app db or owns the connection lifecycle; it hands ordinary
- arguments to each concern (no marshaled plan object) and owns the HTTP decisions.
+ "Orchestrates a transform test run over the module's separated concerns. Callers hand the runner the definitions of the
+ transform and the test; the runner owns the connection lifecycle, hands ordinary arguments to each concern (no
+ marshaled plan object), and owns the HTTP decisions.
- db — app-db reads (transform, database)
+ db — app-db reads (database)
compile — pure: source → SQL + referenced tables (once), then rewrites to temp tables
validator — pure: may this test run? every read faked, every rewritten query on temp tables
executor — the one place warehouse I/O lives: create/drop temp tables, run read-backs
@@ -11,7 +11,7 @@
Flow — compile the source once, then thread that one result so the guard and the rewrite agree:
- resolve (app-db)
+ resolve the database (app-db)
→ compile-source (pure) : the transform's SQL + the tables it reads
→ name + rewrite (pure) : a temp table per input, and the source reading them
→ validate (pure) : reject a test that is incomplete, or whose queries would still
@@ -29,17 +29,21 @@
[metabase-enterprise.transform-testing.expectations.protocol :as expectations.protocol]
[metabase-enterprise.transform-testing.run-tracking :as transform-testing.run-tracking]
[metabase-enterprise.transform-testing.schema :as transform-testing.schema]
+ [metabase-enterprise.transform-testing.util :as transform-testing.u]
[metabase-enterprise.transform-testing.validator :as transform-testing.validator]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
- [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.sql-parsing.core :as sql-parsing]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
@@ -48,6 +52,10 @@
metabase-enterprise.transform-testing.expectations.empty/keep-me
metabase-enterprise.transform-testing.expectations.equals/keep-me)
+(mr/def ::labels
+ "The table the author wrote for each temp table of the run."
+ [:map-of :string :string])
+
(mr/def ::plan
"What [[validated-plan]] hands the run that follows it."
[:map {:closed true}
@@ -55,27 +63,71 @@
[:database ::lib.schema.metadata/database]
[:input->table [:map-of ::transform-testing.schema/input :string]]
[:output-table :string]
- [:labels [:map-of :string :string]]
+ [:labels ::labels]
[:replacements ::transform-testing.compile/table-replacements]
[:compiled ::transform-testing.compile/compiled-query]])
-(defn- table-labels
+(mr/def ::built-expectation
+ "An expectation built by [[expectations.protocol/build]]."
+ [:fn #(satisfies? expectations.protocol/Expectation %)])
+
+(mr/def ::context
+ "What an expectation is given to build its queries and read their answers: the driver, the output temp table and
+ the columns it has, the input-table replacements, and the expectation's own temp tables once they exist."
+ [:map {:closed true}
+ [:driver :keyword]
+ [:output-table :string]
+ [:replacements ::transform-testing.compile/table-replacements]
+ [:output-columns {:optional true} [:sequential ::transform-testing.schema/result-column]]
+ [:temp-tables {:optional true} [:map-of :string :string]]])
+
+(mr/def ::probe
+ "A query an expectation needs run."
+ [:map {:closed true}
+ [:query :string]
+ [:params [:maybe [:sequential ::lib.schema.common/field-value]]]
+ [:max-rows :int]])
+
+(mr/def ::probes
+ "The queries an expectation needs, by probe id: the ids the expectation types use."
+ [:map {:closed true}
+ [:violations {:optional true} ::probe]
+ [:comparison {:optional true} ::probe]
+ [:actual-count {:optional true} ::probe]])
+
+(mr/def ::probe-results
+ "What each probe of [[::probes]] read: its rows and the columns describing them."
+ (let [result [:map {:closed true}
+ [:rows [:sequential [:sequential ms/FieldValue]]]
+ [:columns [:sequential ::transform-testing.schema/result-column]]]]
+ [:map {:closed true}
+ [:violations {:optional true} result]
+ [:comparison {:optional true} result]
+ [:actual-count {:optional true} result]]))
+
+(mu/defn- table-labels :- ::labels
"The map from each temp table of the run to the table the author wrote, for rewriting warehouse error messages that
can only speak in generated names."
- [transform input->table output-table]
+ [transform :- ::transforms.schema/transform.definition
+ input->table :- [:map-of ::transform-testing.schema/input :string]
+ output-table :- :string]
(into {output-table (transform-testing.validator/table-label (select-keys (:target transform) [:schema :name]))}
(map (fn [[input table]] [table (transform-testing.validator/table-label (:table input))]))
input->table))
-(defn- warehouse-said
+(mu/defn- warehouse-said :- [:maybe :string]
"What the warehouse told us about `e`, with every generated temp-table name replaced by the one
the author wrote, or nil when it said nothing."
- [e labels]
+ [e :- (ms/InstanceOfClass Throwable)
+ labels :- ::labels]
(transform-testing.errors/remap-message (ex-message e) labels))
-(defn- rethrow-remapped
+(mu/defn- rethrow-remapped :- :nil
"Rethrow `e` as `error-type` behind `message`, quoting the warehouse where it had something to say."
- [e error-type message labels]
+ [e :- (ms/InstanceOfClass Throwable)
+ error-type :- :keyword
+ message :- :string
+ labels :- ::labels]
(let [cause (warehouse-said e labels)]
(throw (transform-testing.errors/ex
error-type
@@ -83,18 +135,24 @@
{:cause cause}
e))))
-(defn- run-probes!
+(mu/defn- run-probes! :- ::probe-results
"Run one expectation's probes on `conn`, returning `{probe-id rows}`."
- [driver conn probes]
+ [driver :- :keyword
+ conn :- ::transform-testing.schema/connection
+ probes :- ::probes]
(update-vals probes
(fn [{:keys [query params max-rows]}]
(transform-testing.executor/run-query driver conn [query (vec params)] max-rows))))
-(defn- check-expectation
+(mu/defn- check-expectation :- ::transform-testing.schema/expectation-result
"The result map for one expectation. An expectation whose own query fails is reported as an error
on that expectation rather than failing the whole run. The temp tables it needs are created
before its probes and dropped after them."
- [driver conn context expectation labels]
+ [driver :- :keyword
+ conn :- ::transform-testing.schema/connection
+ context :- ::context
+ expectation :- ::built-expectation
+ labels :- ::labels]
(let [temp-tables (atom {})]
(try
(doseq [[id query] (expectations.protocol/temp-tables expectation context)]
@@ -116,20 +174,18 @@
(doseq [table (vals @temp-tables)]
(transform-testing.executor/drop-temp-table! driver conn table))))))
-(defn- testable-transform
- "The transform `transform-id` names, if it is one a test can run."
- [transform-id]
- (let [transform (api/check-404 (transform-testing.db/transform transform-id))]
- (when-not (transforms-base.u/query-transform? transform)
- (throw (transform-testing.errors/ex
- ::transform-testing.errors/unsupported-transform
- (tru "Only query transforms can be tested.")
- {:transform-id transform-id})))
- transform))
+(mu/defn- check-testable-transform :- :nil
+ "Refuse `transform` unless it is one a test can run."
+ [transform :- ::transforms.schema/transform.definition]
+ (when-not (transforms-base.u/query-transform? transform)
+ (throw (transform-testing.errors/ex
+ ::transform-testing.errors/unsupported-transform
+ (tru "Only query transforms can be tested.")
+ {}))))
-(defn- testable-database
+(mu/defn- testable-database :- ::lib.schema.metadata/database
"The database `transform` reads, if its driver can run a transform test."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(let [database-id (transforms-base.u/transform-source-database transform)
_ (api/check-404 (some-> database-id transform-testing.db/database-exists?))
database (driver.u/lib-database database-id)
@@ -142,9 +198,10 @@
{:driver driver})))
database))
-(defn- parsed-source
+(mu/defn- parsed-source :- ::transform-testing.compile/compiled-source
"The transform's source compiled to SQL, with the tables it reads, or a refusal naming what would not parse."
- [driver transform transform-id]
+ [driver :- :keyword
+ transform :- ::transforms.schema/transform.definition]
(try
(transform-testing.compile/compile-source driver transform)
(catch Exception e
@@ -153,12 +210,15 @@
::transform-testing.errors/unparseable-source
(tru "The transform source SQL could not be parsed for test input validation: {0}"
(sql-parsing/parse-error-message e))
- {:transform-id transform-id}))
+ {}))
(throw e)))))
-(defn- create-inputs!
+(mu/defn- create-inputs! :- :nil
"Materialize each input of `input->table` as its temp table on `conn`."
- [driver conn input->table labels]
+ [driver :- :keyword
+ conn :- ::transform-testing.schema/connection
+ input->table :- [:map-of ::transform-testing.schema/input :string]
+ labels :- ::labels]
(doseq [[input table] input->table]
(try
(transform-testing.executor/create-temp-table!
@@ -169,19 +229,28 @@
(transform-testing.validator/table-label (:table input)))
labels)))))
-(defn- create-output!
+(mu/defn- create-output! :- :nil
"Run the transform under test on `conn`, materializing its output as the temp table `output-table`."
- [driver conn output-table compiled-transform labels]
+ [driver :- :keyword
+ conn :- ::transform-testing.schema/connection
+ output-table :- :string
+ compiled-transform :- ::transform-testing.compile/compiled-query
+ labels :- ::labels]
(try
(transform-testing.executor/create-temp-table! driver conn output-table compiled-transform)
+ nil
(catch Exception e
(rethrow-remapped e ::transform-testing.errors/transform-failed
(tru "The transform under test failed to run:")
labels))))
-(defn- check-expectations
+(mu/defn- check-expectations :- [:sequential ::transform-testing.schema/expectation-result]
"The result of every expectation against the output temp table, passing ones included."
- [driver conn context expectations labels]
+ [driver :- :keyword
+ conn :- ::transform-testing.schema/connection
+ context :- ::context
+ expectations :- ::transform-testing.schema/expectations
+ labels :- ::labels]
(let [context (assoc context
:output-columns
(transform-testing.executor/table-columns driver conn (:output-table context)))]
@@ -197,13 +266,13 @@
Throws the run's typed refusals — an untestable transform or driver, SQL that will not parse, a test the
validator rejects — so a caller that only wants the answer to \"could this run?\" gets it by asking for the plan
and discarding it."
- [transform-id :- ::lib.schema.id/transform
+ [transform :- ::transforms.schema/transform.definition
inputs :- ::transform-testing.schema/inputs
expectations :- ::transform-testing.schema/expectations]
- (let [transform (testable-transform transform-id)
- database (testable-database transform)
+ (check-testable-transform transform)
+ (let [database (testable-database transform)
driver (keyword (:engine database))
- source (parsed-source driver transform transform-id)
+ source (parsed-source driver transform)
input->table (into {} (map (fn [input] [input (driver/temp-table-name driver)])) inputs)
output-table (driver/temp-table-name driver)
default-schema (:default-schema database)
@@ -225,27 +294,30 @@
:replacements replacements
:compiled compiled}))
-(mu/defn validate-transform-test :- :nil
- "Refuse a test of the transform `transform-id` with these `inputs` and `expectations` unless it could run: the same
+(mu/defn validate-test :- :nil
+ "Refuse a test of `transform` with these `inputs` and `expectations` unless it could run: the same
checks a run makes, up to but not including the connection.
Returns nil when there is nothing to refuse, and otherwise throws the refusal the run would have thrown, so a test
is turned away as it is authored rather than only when somebody runs it."
- [transform-id :- ::lib.schema.id/transform
+ [transform :- ::transforms.schema/transform.definition
inputs :- ::transform-testing.schema/inputs
expectations :- ::transform-testing.schema/expectations]
- (validated-plan transform-id inputs expectations)
+ (validated-plan transform inputs expectations)
nil)
-(mu/defn run-transform-test! :- ::transform-testing.schema/run-result
- "Run the transform test `transform-test` against temp tables and report what each expectation found.
+(mu/defn run-test! :- ::transform-testing.schema/run-result
+ "Run `transform-test` against `transform` on temp tables and report what each expectation found, recording the run
+ under `transform-test-id`.
Throws a typed refusal from [[metabase-enterprise.transform-testing.errors]] when the run cannot happen. A
failing expectation is not a refusal: it rides back as that expectation's own result."
- [{:keys [id transform_id inputs expectations]} :- ::transform-testing.schema/transform-test]
+ [transform :- ::transforms.schema/transform.definition
+ {:keys [inputs expectations]} :- ::transform-testing.schema/transform-test.definition
+ transform-test-id :- ms/PositiveInt]
(let [{:keys [driver database input->table output-table labels replacements compiled]}
- (validated-plan transform_id inputs expectations)
- {run-id :id} (transform-testing.run-tracking/start-run! id api/*current-user-id*)
+ (validated-plan transform inputs expectations)
+ {run-id :id} (transform-testing.run-tracking/start-run! transform-test-id api/*current-user-id*)
final-status (atom :error)]
(try
(let [result
@@ -270,3 +342,10 @@
result)
(finally
(transform-testing.run-tracking/finish-run! run-id @final-status)))))
+
+(mu/defn run-saved-test! :- ::transform-testing.schema/run-result
+ "Run the saved `transform-test` of the saved `transform`, as [[run-test!]] runs definitions, recording the run under
+ the test."
+ [transform :- ::transforms.schema/transform
+ transform-test :- ::transform-testing.schema/transform-test]
+ (run-test! (transforms.u/->definition transform) (transform-testing.u/->definition transform-test) (:id transform-test)))
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
index 99f76e3ccf8..dc24729aa82 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
@@ -6,13 +6,14 @@
[malli.core :as mc]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
-(defn- unknown-dispatch
+(mu/defn- unknown-dispatch :- fn?
"An `:error/fn` for a `:multi` dispatching on `dispatch-key`: says what the value named and what the schema
knows, where Malli would say only \"invalid dispatch value\"."
- [dispatch-key]
+ [dispatch-key :- :keyword]
(fn [{:keys [schema value]} _options]
(let [known (str/join ", " (sort (map (comp name first) (mc/children schema))))]
(if (map? value)
@@ -136,30 +137,35 @@
(let [names (map :name expectations)]
(or (empty? names) (apply distinct? names))))]])
-(mr/def ::transform-test
- "A saved transform test: the transform it covers, the inputs that stand in for its source
- tables, and the expectations checked against its output."
- [:map {:closed true}
- [:id ms/PositiveInt]
- [:entity_id :string]
- [:transform_id ::lib.schema.id/transform]
- [:creator_id ::lib.schema.id/user]
- [:name :string]
- [:description [:maybe :string]]
- [:inputs ::inputs]
- [:expectations ::expectations]
- [:created_at ms/TemporalInstant]
- [:updated_at ms/TemporalInstant]])
-
-(mr/def ::transform-test.create
- "What a client sends to create a transform test."
+(mr/def ::transform-test.definition
+ "A transform test as a client writes it, without the transform it covers."
[:map {:closed true}
- [:transform_id ::lib.schema.id/transform]
[:name ::lib.schema.common/non-blank-string]
[:description {:optional true} [:maybe :string]]
[:inputs ::inputs]
[:expectations ::expectations]])
+(mr/def ::transform-test.create
+ "What a client sends to create a transform test."
+ [:merge
+ [:map {:closed true}
+ [:transform_id ::lib.schema.id/transform]]
+ ::transform-test.definition])
+
+(mr/def ::transform-test
+ "A saved transform test: the transform it covers, the inputs that stand in for its source
+ tables, and the expectations checked against its output."
+ [:merge
+ ::transform-test.definition
+ [:map {:closed true}
+ [:id ms/PositiveInt]
+ [:entity_id :string]
+ [:transform_id ::lib.schema.id/transform]
+ [:creator_id ::lib.schema.id/user]
+ [:description [:maybe :string]]
+ [:created_at ms/TemporalInstant]
+ [:updated_at ms/TemporalInstant]]])
+
(mr/def ::transform-test.update
"What a client sends to update a transform test. Omitted fields are left unchanged."
[:map {:closed true}
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/util.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/util.clj
index e944f847045..cdade7d43c6 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/util.clj
@@ -1,15 +1,22 @@
(ns metabase-enterprise.transform-testing.util
"Helpers shared across the module."
(:require
- [metabase.util :as u]))
+ [metabase-enterprise.transform-testing.schema :as transform-testing.schema]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
-(defn fold-identifier
+(mu/defn fold-identifier :- [:maybe :string]
"`identifier` — a schema or a table name — as this module compares it, or nil when there is none.
Every comparison here ignores case: a reference against a declared input, a declared schema against the database's
default one, and two declared inputs against each other. That is how the rewrite matches, and how an engine reads an
unquoted identifier."
- [identifier]
+ [identifier :- [:maybe :string]]
(some-> identifier u/lower-case-en))
+
+(mu/defn ->definition :- ::transform-testing.schema/transform-test.definition
+ "What the saved `transform-test` checks, as a client writes it."
+ [transform-test :- ::transform-testing.schema/transform-test]
+ (select-keys transform-test [:name :description :inputs :expectations]))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_inspector/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_inspector/api.clj
index b81ca9519a5..5c5fb6ecead 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_inspector/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_inspector/api.clj
@@ -19,6 +19,7 @@
[metabase.tracing.core :as tracing]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.core :as transforms.core]
+ [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.malli.schema :as ms]))
@@ -64,7 +65,7 @@
Returns structural metadata and available lens types."
[{:keys [id]} :- [:map {:closed true} [:id ms/PositiveInt]]]
(let [transform (api/read-check :model/Transform id)
- _ (transforms.core/check-feature-enabled! transform)
+ _ (transforms.core/check-feature-enabled! (transforms.u/->definition transform))
result (tracing/with-span :transforms "transforms.inspector.discover"
{:transform/id id
:transform/source-type (name (transforms-base.u/transform-source-type (:source transform)))}
@@ -90,7 +91,7 @@
[:lens-id ms/NonBlankString]]
params :- ::inspector.schema/lens-params.request]
(let [transform (api/read-check :model/Transform id)
- _ (transforms.core/check-feature-enabled! transform)
+ _ (transforms.core/check-feature-enabled! (transforms.u/->definition transform))
result (tracing/with-span :transforms "transforms.inspector.lens"
{:transform/id id
:transform/source-type (name (transforms-base.u/transform-source-type (:source transform)))
@@ -125,7 +126,7 @@
[:query ::lib-be.schema/maybe-legacy-query]
[:lens_params {:optional true} [:maybe ::inspector.schema/lens-params.request]]]]
(let [transform (api/read-check :model/Transform id)]
- (transforms.core/check-feature-enabled! transform)
+ (transforms.core/check-feature-enabled! (transforms.u/->definition transform))
(let [info {:executed-by api/*current-user-id*
:context :transform-inspector
:transform-id id
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index b05c1d8787a..c7f0036bfe2 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -245,7 +245,7 @@
:definition (-> (lib/query mp products)
(lib/aggregate (lib/sum price)))}
:model/Transform _ {:name "Transform on products"
- :source {:type "query" :query (lib/query mp products)}
+ :source {:type :query :query (lib/query mp products)}
:target {:type "table" :name (mt/random-name)}}]
(events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})
@@ -447,8 +447,9 @@
:model/Transform {transform-id :id} {:name "Transform - inactivetabletest"
:source {:type :query
:query (lib/query mp products)}
- :target {:schema "PUBLIC"
- :name "inactive_transform_table"}}]
+ :target {:type "table"
+ :schema "PUBLIC"
+ :name "inactive_transform_table"}}]
;; Simulate the transform having been run: link it to its (now inactive) target table
(t2/update! :model/Transform :id transform-id {:target_table_id table-id})
(deps.hooks/after-transform-run transform-id table-id)
@@ -1246,13 +1247,15 @@
:model/Transform {unreffed-transform-id :id} {:name "Unreferenced Transform - unreftest"
:source {:type :query
:query (lib/query mp products)}
- :target {:schema "PUBLIC"
- :name "unreferenced_transform_table"}}
+ :target {:type "table"
+ :schema "PUBLIC"
+ :name "unreferenced_transform_table"}}
:model/Transform {referenced-transform-id :id} {:name "Referenced Transform - unreftest"
:source {:type :query
:query (lib/query mp products)}
- :target {:schema "PUBLIC"
- :name "referenced_transform_table"}}]
+ :target {:type "table"
+ :schema "PUBLIC"
+ :name "referenced_transform_table"}}]
;; Simulate the referenced transform having been run: link it to its target table
;; so the table→transform dep is visible in the dependency graph.
(t2/update! :model/Transform :id referenced-transform-id {:target_table_id referenced-table-id})
@@ -2821,13 +2824,13 @@
:owner_user_id (mt/user->id :crowberto)
:source {:type :query
:query (lib/query mp products)}
- :target {:schema "PUBLIC"
+ :target {:type "table" :schema "PUBLIC"
:name "user_owned_transform_table"}}
:model/Transform {transform2-id :id} {:name "Email Owned Transform - ownertest"
:owner_email "[email protected]"
:source {:type :query
:query (lib/query mp products)}
- :target {:schema "PUBLIC"
+ :target {:type "table" :schema "PUBLIC"
:name "email_owned_transform_table"}}]
(deps.test/synchronously-run-dependency-job!)
(let [response (mt/user-http-request :crowberto :get 200
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
index fbc4d7f03f1..a59f748de4d 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
@@ -369,36 +369,3 @@
(serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
(is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))
(is (seq @submitted)))))))
-
-(deftest ^:synchronized database-delete-marks-source-transforms-stale-test
- (testing "deleting a database marks its source transforms' analysis findings stale so they surface on /dependency-diagnostics/broken (GDGT-2447)"
- (deps.test/run-with-dependencies-setup!
- (fn [_mp]
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Transform {transform-id :id}
- {:source {:type :query
- :query {:database db-id
- :type :native
- :native {:query "SELECT 1"}}}
- :target {:schema "public" :name "out" :type :table}}
- :model/AnalysisFinding _ {:entity_type :transform
- :entity_id transform-id
- :result true
- :analyzed_at :%now}]
- (t2/delete! :model/Database :id db-id)
- (is (nil? (t2/select-one :model/Database :id db-id))
- "the Database row is actually gone")
- (is (pending? :transform transform-id))
- (deps.test/synchronously-run-dependency-job!)
- (testing "and the analysis finding job flips :result false so the transform shows on broken-diagnostics"
- (deps.test/run-analysis-finding-job!)
- (is (false? (t2/select-one-fn :result :model/AnalysisFinding
- :entity_type :transform
- :entity_id transform-id)))
- (testing "an analysis_finding_error row points the transform at itself so the diagnostics page (`/graph/breaking`) surfaces it"
- (is (=? {:source_entity_type :transform
- :source_entity_id transform-id
- :error_type :validation-exception-error}
- (t2/select-one :model/AnalysisFindingError
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index 63269fcbec3..71cf62b67b7 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -7,6 +7,7 @@
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.graph.core :as graph]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.queries.models.card :as card]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -204,7 +205,7 @@
(mt/with-temp [:model/Transform transform]
(is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform))))
(computed! :transform (:id transform))
- (t2/update! :model/Transform :id (:id transform) {:source {:type "query" :query (mt/mbql-query products)}})
+ (t2/update! :model/Transform :id (:id transform) {:source {:type :query :query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}})
(is (pending? :transform (:id transform)))))
(testing "snippets"
(mt/with-temp [:model/NativeQuerySnippet snippet]
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
index 0c883bf04f0..a94845e133a 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/dependency_test.clj
@@ -101,9 +101,9 @@
(mt/with-premium-features #{}
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
:model/Transform {transform-id :id} {:name "Test Transform"
- :source {:type "query"
- :query (mt/mbql-query nil
- {:source-table (str "card__" card-id)})}
+ :source {:type :query
+ :query (let [mp (lib-be/application-database-metadata-provider (mt/id))]
+ (lib/query mp (lib.metadata/card mp card-id)))}
:target {:type "table" :name (mt/random-name)}}]
(forget-status! :card card-id)
(forget-status! :transform transform-id)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
index 124ca24e10f..163bbe2ceec 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/collection_api_test.clj
@@ -9,6 +9,7 @@
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.collections.models.collection :as collection]
[metabase.events.core :as events]
+ [metabase.lib.core :as lib]
[metabase.remote-sync.test-util :as remote-sync.tu]
[metabase.test :as mt]
[metabase.util :as u]
@@ -185,8 +186,8 @@
transform-id (t2/insert-returning-pk! :model/Transform
{:name "Transform In Folder"
:collection_id (:id folder)
- :source {:type "query"
- :query (mt/native-query {:query "SELECT 1"})}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table" :schema "PUBLIC" :name "ghy_4399"}})
transform (t2/select-one :model/Transform :id transform-id)
_ (events/publish-event! :event/transform-create
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
index 37fcd7248a0..7ff92e689b3 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/permissions_test.clj
@@ -14,6 +14,12 @@
(use-fixtures :each remote-sync.tu/allow-remote-synced-collections)
+(defn- venues-query
+ "Build an MBQL 5 query over the venues table."
+ []
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id :venues)))))
+
(defn- text->prose-mirror-ast
"Convert text to a simple ProseMirror AST for document content."
[text]
@@ -704,10 +710,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -723,10 +727,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -744,10 +746,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -765,10 +765,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -787,10 +785,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -808,10 +804,8 @@
:namespace :transforms}]
(is (false? (mi/can-create? :model/Transform {:name "New Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -828,10 +822,8 @@
:namespace :transforms}]
(is (true? (mi/can-create? :model/Transform {:name "New Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -847,10 +839,8 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Test Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
@@ -867,7 +857,7 @@
:namespace :transforms}
:model/Transform {transform-id :id} {:name "Read-only Transform"
:collection_id coll-id
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:database (mt/id)
:type "table"
@@ -891,10 +881,8 @@
:namespace :transforms}]
(is (false? (mi/can-create? :model/Transform {:name "New Transform"
:collection_id coll-id
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :venues)}}}
+ :source {:type :query
+ :query (venues-query)}
:target {:database (mt/id)
:type "table"
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
index 21c9457297d..f8be35165ed 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
@@ -126,7 +126,7 @@
:name "Grandchild via Native"}
:model/Transform {transform-id :id}
- {:source {:type "query"
+ {:source {:type :query
:query (lib/query mp (lib.metadata/card mp old-id))}
:name "acceptance_transform"
:target {:database (mt/id) :type "table" :name "acceptance_transform"}}
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj
index 53384c76b5b..8b1b2c702ea 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj
@@ -291,7 +291,7 @@
lib/append-stage
(lib/filter (lib/= (lib.metadata/field mp (mt/id :orders :id)) 1)))]
(mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:database (mt/id) :type "table" :name "out"}}]
(replacement.field-refs/upgrade-field-refs! [:transform transform-id])
(is (=? {:source {:type :query
@@ -305,28 +305,16 @@
(let [mp (mt/metadata-provider)
query (lib/native-query mp "SELECT * FROM orders")]
(mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:database (mt/id) :type "table" :name "out"}}]
(is (nil? (replacement.field-refs/upgrade-field-refs! [:transform transform-id])))))))
-(deftest transform-upgrade-field-refs!-broken-query-test
- (testing "should not crash on a transform with a broken query"
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query" :query (lib/native-query mp "SELECT 1")}
- :target {:database (mt/id) :type "table" :name "out"}}]
- ;; simulate a broken query by updating source directly in the DB
- (t2/query-one {:update :transform
- :set {:source "{\"type\":\"query\",\"query\":{}}"}
- :where [:= :id transform-id]})
- (is (nil? (replacement.field-refs/upgrade-field-refs! [:transform transform-id])))))))
-
(deftest transform-upgrade-field-refs!-no-changes-test
(testing "should not update a transform when there are no changes"
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
(mt/with-temp [:model/Transform {transform-id :id, updated-at :updated_at} {:name "test transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:database (mt/id) :type "table" :name "out"}}]
(replacement.field-refs/upgrade-field-refs! [:transform transform-id])
(is (= updated-at (:updated_at (t2/select-one :model/Transform :id transform-id))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
index 2c62d55e158..64b7784c08f 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
@@ -157,7 +157,7 @@
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
(mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:database (mt/id) :type "table" :name "out"}}]
(replacement.field-refs/upgrade-field-refs! [:transform transform-id])
(replacement.source-swap/swap-source! [:transform transform-id]
@@ -167,22 +167,6 @@
(get-in (t2/select-one :model/Transform :id transform-id)
[:source :query :stages 0 :source-table])))))))))
-(deftest transform-swap-source!-broken-query-test
- (testing "swap-source! should not crash on a transform with a broken query"
- (mt/with-premium-features #{:dependencies}
- (mt/with-test-user :rasta
- (let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query"
- :query (lib/native-query mp "SELECT 1")}
- :target {:database (mt/id) :type "table" :name "out"}}]
- (t2/query-one {:update :transform
- :set {:source "{\"type\":\"query\",\"query\":{}}"}
- :where [:= :id transform-id]})
- (is (nil? (replacement.source-swap/swap-source! [:transform transform-id]
- [:table (mt/id :orders)]
- [:table (mt/id :reviews)])))))))))
-
;;; ------------------------------------------------- Segment ----------------------------------------------------------
(deftest segment-swap-source!-source-table-test
@@ -502,7 +486,7 @@
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
(mt/with-temp [:model/Transform {transform-id :id} {:name "test transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:database (mt/id) :type "table" :name "out"}}]
(replacement.field-refs/upgrade-field-refs! [:transform transform-id])
(mt/with-model-cleanup [:model/Dependency]
diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
index 2cee74007f7..cfd2820685c 100644
--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
@@ -7,6 +7,7 @@
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.search.scoring :as ee-scoring]
+ [metabase.lib.core :as lib]
[metabase.search.appdb.scoring-test :as appdb.scoring-test]
[metabase.search.in-place.scoring :as scoring]
[metabase.search.test-util :as search.tu]
@@ -212,8 +213,8 @@
(mt/with-temp [:model/Card {c1 :id} {}
:model/Card {c2 :id} {}
:model/Transform {t1 :id} {:name "test transform"
- :source {:type "query"
- :query (mt/native-query {:query "SELECT 1"})}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name (mt/random-name)}}
:model/RecentViews _ (recent-view c1 now)]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 52ab4f4208f..ccd4a19e79d 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -1262,45 +1262,6 @@
:table_id (:id table)}
(t2/select-one :model/Card :name "Customers")))))))))
-(deftest orphaned-transform-yaml-round-trip-test
- (testing "A Transform whose source database was deleted round-trips through YAML storage as a tombstone"
- (mt/with-premium-features #{:transforms-basic}
- (ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (ts/with-dbs [source-db dest-db]
- (ts/with-db source-db
- (t2/delete! :model/TransformTag)
- (let [db (ts/create! :model/Database :name "soon-to-be-deleted")
- table (ts/create! :model/Table :name "customers" :db_id (:id db))
- mp (lib-be/application-database-metadata-provider (:id db))]
- (ts/create! :model/Transform
- :name "Orphan Transform"
- :source {:type "query"
- :query (lib/query mp (lib.metadata/table mp (:id table)))}
- :target {:database (:id db)
- :type "table"
- :schema "public"
- :name "orphan_target"})
- (t2/delete! :model/Database :id (:id db))
- (storage/store! (serdes/with-cache (into [] (extract/extract {})))
- (storage.files/file-writer dump-dir))))
- (let [file (io/file dump-dir "collections" "transforms" "orphan_transform.yaml")
- exported (yaml/from-file file)]
- (testing "the exported query carries no metadata provider"
- (is (=? {:database nil, :lib/type "mbql/query"} (get-in exported [:source :query])))
- (is (not (contains? (get-in exported [:source :query]) :lib/metadata))))
- (dump/spit-yaml! file (assoc-in exported [:source :query :lib/metadata] nil)))
- (ts/with-db dest-db
- (t2/delete! :model/TransformTag)
- (is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
- "an export written with a nil `lib/metadata` still loads")
- (let [transform (t2/select-one :model/Transform :name "Orphan Transform")]
- (is (=? {:source_database_id nil
- :source {:type :query}}
- transform))
- (is (=? {"database" nil
- "stages" [{"source-table" pos-int?}]}
- (get-in transform [:source :query]))))))))))
-
(deftest transform-test-round-trip-test
(testing "A transform test is exported under its transform's path and imported with its transform"
(mt/with-premium-features #{:transforms-basic}
@@ -1314,8 +1275,8 @@
:name "Orders Summary"
:collection_id (:id coll)
:creator_id (:id creator)
- :source {:type "query"
- :query {:database (:id db) :type "native" :native {:query "SELECT 1 AS ID"}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider (:id db)) "SELECT 1 AS ID")}
:target {:database (:id db) :type "table" :schema "PUBLIC" :name "orders_summary"})]
(ts/create! :model/TransformTest
:transform_id (:id transform)
@@ -1381,8 +1342,8 @@
:name "Revenue Report"
:collection_id (:id coll)
:creator_id (:id creator)
- :source {:type "query"
- :query {:database (:id db) :type "native" :native {:query "SELECT 1 AS ID"}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider (:id db)) "SELECT 1 AS ID")}
:target {:database (:id db) :type "table" :schema "PUBLIC" :name "revenue"})]
(ts/create! :model/TransformTest
:transform_id (:id transform)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 8e1286c3550..65ffe1ec84f 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -2306,10 +2306,9 @@
:description "A test transform for serialization"
:entity_id "2HzIFwJ6720JAx07UMavl"
:collection_id coll-id
- :source {:query {:database db-id
- :type "query"
- :query {:source-table table-id}}
- :type "query"}
+ :source {:type :query
+ :query (let [mp (lib-be/application-database-metadata-provider db-id)]
+ (lib/query mp (lib.metadata/table mp table-id)))}
:target {:database db-id
:type "table"
:schema "public"
@@ -2323,7 +2322,7 @@
:entity_id "pythonEtlPythonEtlxxx"
:collection_id coll-id
:source_database_id db-id
- :source {:type "python"
+ :source {:type :python
:source-database db-id
:source-tables [{:alias "orders"
:table_id table-id}]
@@ -2392,47 +2391,6 @@
(is (= #{transform-eid python-transform-eid}
(ids-by-model "Transform" (extract/extract {}))))))))))
-(deftest transform-with-null-source-database-extract-test
- (testing "A transform whose source database has been deleted serializes as a tombstone (GDGT-2447)"
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-empty-h2-app-db!
- (ts/with-temp-dpc [:model/Database {db-id :id} {:name "Soon-to-be-deleted DB"}
- :model/Transform {transform-id :id transform-eid :entity_id}
- {:name "Orphan Transform"
- :entity_id "orphanXxxxxxxxxxxxxxx"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"}}}
- :target {:database db-id
- :type "table"
- :schema "public"
- :name "orphan_target"}}]
- ;; Delete the database — ON DELETE SET NULL nulls source_database_id and target_db_id.
- (t2/delete! :model/Database :id db-id)
- (let [reloaded (t2/hydrate (t2/select-one :model/Transform :id transform-id) :tags :indexes)
- ser (serdes/extract-one "Transform" {} reloaded)]
- (is (nil? (:source_database_id reloaded))
- "Database deletion should have nulled the column")
- (testing "exported entity carries a nil source_database_id"
- (is (=? {:serdes/meta [{:model "Transform" :id transform-eid}]
- :name "Orphan Transform"}
- ser))
- (is (nil? (:source_database_id ser))))
- (testing "exported source is marked :serdes/unresolved with the dead :database ref nulled"
- ;; The :database slot is nulled because the numeric id refers to a now-deleted database —
- ;; keeping it would make the destination instance's before-insert hook bind
- ;; source_database_id to a stale id and crash the FK.
- (is (true? (get-in ser [:source :serdes/unresolved])))
- (is (nil? (get-in ser [:source :query :database]))))
- (testing "the native SQL body is preserved verbatim as a breadcrumb"
- ;; `lib-be/normalize-query` rewrites the raw query to MBQL5 (`:stages [...]`) at read time,
- ;; so the native text now lives under `:stages [0] :native`.
- (is (= "SELECT 1" (get-in ser [:source :query :stages 0 :native]))))
- (testing "Transform/dependencies does not emit a Database dep when source_database_id is nil"
- (is (not-any? #(some (fn [{:keys [model]}] (= "Database" model)) %)
- (serdes/deserialization-dependencies (assoc reloaded :tags [])))))))))))
-
(deftest table-with-transform-id-dependency-test
(testing "Table created by a Transform declares the Transform as a serdes dependency (GDGT-2444)"
(mt/with-premium-features #{:transforms-basic}
@@ -2445,10 +2403,8 @@
transform-eid :entity_id}
{:name "Transform Creating Table"
:entity_id "transformCreatingTblx"
- :source {:query {:database db-id
- :type "native"
- :native {:query "select 1 as x"}}
- :type "query"}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "select 1 as x")}
:target {:database db-id
:type "table"
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index d873c4fad5e..20af09026c7 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -2775,8 +2775,8 @@
:name "Test Transform"
:description "A test transform"
:collection_id (:id coll)
- :source {:query (mbql5-query (:id db) (:id table))
- :type "query"}
+ :source {:type :query
+ :query (mbql5-query (:id db) (:id table))}
:target {:database (:id db)
:type "table"
:schema "public"
@@ -2796,6 +2796,37 @@
(is (some? transform))
(is (= (:id db) (:source_database_id transform)))))))))))
+(deftest transform-unresolved-source-skipped-test
+ (testing "Transform exported without its deleted source database is skipped on load"
+ (mt/with-premium-features #{:transforms-basic}
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (t2/delete! :model/TransformTag)
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))]
+ (ts/create! :model/Transform
+ :name "Unresolved Transform"
+ :source {:type :query
+ :query (mbql5-query (:id db) (:id table))}
+ :target {:database (:id db)
+ :type "table"
+ :schema "public"
+ :name "target_table"})
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+ (let [unresolved (mapv (fn [entity]
+ (cond-> entity
+ (= "Transform" (-> entity :serdes/meta last :model))
+ (update :source #(-> %
+ (assoc :serdes/unresolved true)
+ (assoc-in [:query :database] nil)))))
+ @serialized)]
+ (ts/with-db dest-db
+ (t2/delete! :model/TransformTag)
+ (serdes.load/load-metabase! (ts/ingestion-in-memory unresolved))
+ (is (not (t2/exists? :model/Transform :name "Unresolved Transform")))
+ (is (t2/exists? :model/Database :name "my-db")))))))))
+
(deftest transform-checkpoint-field-remap-test
(testing "checkpoint-filter-field-id survives serdes into an instance with different field IDs (GDGT-2906)"
(mt/with-premium-features #{:transforms-basic}
@@ -2809,7 +2840,7 @@
:base_type :type/DateTime)
_ (ts/create! :model/Transform
:name "Checkpoint Transform"
- :source {:type "query"
+ :source {:type :query
:query (mbql5-query (:id db) (:id table))
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id (:id field)
@@ -2853,7 +2884,7 @@
table (ts/create! :model/Table :name "customers" :db_id (:id db))
_ (ts/create! :model/Transform
:name "Stale Checkpoint Transform"
- :source {:type "query"
+ :source {:type :query
:query (mbql5-query (:id db) (:id table))
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id 999999999}}
@@ -2875,55 +2906,6 @@
[:source :source-incremental-strategy :checkpoint-filter-field-id]))
"the stale numeric id survives the import unchanged")))))))
-(deftest transform-with-deleted-source-database-load-test
- (testing "An orphaned transform (source database was deleted before export) round-trips through serdes as a tombstone (GDGT-2447)"
- (mt/with-premium-features #{:transforms-basic}
- (let [serialized (atom nil)]
- (ts/with-dbs [source-db dest-db]
- (ts/with-db source-db
- (t2/delete! :model/TransformTag)
- (let [db (ts/create! :model/Database :name "soon-to-be-deleted")
- table (ts/create! :model/Table :name "customers" :db_id (:id db))
- coll (ts/create! :model/Collection :name "Transform Collection" :namespace :transforms)
- _ (ts/create! :model/Transform
- :name "Orphan Transform"
- :description "Transform whose source DB will be deleted"
- :collection_id (:id coll)
- :source {:query (mbql5-query (:id db) (:id table))
- :type "query"}
- :target {:database (:id db)
- :type "table"
- :schema "public"
- :name "orphan_target"})]
- ;; Deleting the database cascade-SET-NULLs source_database_id (FK action)
- ;; AND cascade-deletes the Table (Field FKs) — the transform survives as a tombstone.
- (t2/delete! :model/Database :name "soon-to-be-deleted")
- (reset! serialized (into [] (serdes.extract/extract {})))))
- (let [minimal (mapv (fn [entity]
- (if (= "Transform" (-> entity :serdes/meta last :model))
- (select-keys entity [:serdes/meta :entity_id :name
- :source :target])
- entity))
- @serialized)]
- (testing "the extracted transform carries the :serdes/unresolved flag and verbatim body"
- (let [extracted (first (filter #(= "Transform" (-> % :serdes/meta last :model)) minimal))]
- (is (some? extracted))
- (is (true? (get-in extracted [:source :serdes/unresolved])))
- ;; native query text is preserved verbatim
- (is (some? (get-in extracted [:source :query])))))
- (ts/with-db dest-db
- (t2/delete! :model/TransformTag)
- (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
- (let [transform (t2/select-one :model/Transform :name "Orphan Transform")]
- (testing "transform round-trips and ends up as a tombstone in the destination instance"
- (is (some? transform))
- (is (nil? (:source_database_id transform))
- "source_database_id should be nil — there's no DB to bind to")
- (is (not (contains? (:source transform) :serdes/unresolved))
- "the :serdes/unresolved marker should have been stripped on import")
- (is (some? (get-in transform [:source :query]))
- "the query body should be preserved as a breadcrumb"))))))))))
-
(deftest table-created-by-transform-load-test
(testing "Table created by a Transform can be imported via serialization (GDGT-2444)"
(mt/with-premium-features #{:transforms-basic}
@@ -2934,10 +2916,9 @@
(let [db (ts/create! :model/Database :name "my-db")
transform (ts/create! :model/Transform
:name "Hello Transform"
- :source {:query {:database (:id db)
- :type "native"
- :native {:query "select 'hello' message"}}
- :type "query"}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider (:id db))
+ "select 'hello' message")}
:target {:database (:id db)
:type "table"
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/api_e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/api_e2e_test.clj
index 25215efdab6..03b6a336ed6 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/api_e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/api_e2e_test.clj
@@ -25,7 +25,7 @@
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))]
(mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." table))}
:target {:type "table"
:schema schema
@@ -108,7 +108,10 @@
(with-transforms-enabled #{:transforms-basic :transforms-python :transforms-testing}
(testing "422 — the environment prevents a run: not a query transform"
(mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "python" :source-database (mt/id)}}
+ {:source {:type :python
+ :source-database (mt/id)
+ :source-tables [{:alias "people" :table_id (mt/id :people)}]
+ :body "def transform(people):\n return people"}}
:model/TransformTest {test-id :id} {:transform_id transform-id}]
(let [response (mt/user-http-request :crowberto :post 422
(format "ee/transform-test/%d/run" test-id))]
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/api_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/api_test.clj
index dd8b9f4c67d..f9bf7c0e66c 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/api_test.clj
@@ -1,7 +1,11 @@
(ns metabase-enterprise.transform-testing.api-test
(:require
[clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.permissions.test-util :as perms.tu]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -18,7 +22,7 @@
"A transform reading the table `inputs` stands in for, writing the table `expectations` reads, so that a test of
it is one the write endpoints accept."
[]
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT ID, NAME FROM PUBLIC.PEOPLE")}
:target {:type "table" :schema "PUBLIC" :name "PEOPLE_SUMMARY" :database (mt/id)}})
@@ -133,15 +137,15 @@
(testing "expectations need a name"
(mt/user-http-request :crowberto :post 400 "ee/transform-test"
(assoc valid :expectations [{:type "empty" :sql "SELECT 1"}])))
- (testing "the transform must exist"
- (mt/user-http-request :crowberto :post 404 "ee/transform-test" (assoc valid :transform_id Integer/MAX_VALUE))
- (mt/user-http-request :crowberto :put 404 (str "ee/transform-test/" test-id) {:transform_id Integer/MAX_VALUE}))))))
+ (testing "transform must exist"
+ (mt/user-http-request :crowberto :post 403 "ee/transform-test" (assoc valid :transform_id Integer/MAX_VALUE))
+ (mt/user-http-request :crowberto :put 403 (str "ee/transform-test/" test-id) {:transform_id Integer/MAX_VALUE}))))))
(deftest run-transform-test-endpoint-test
(mt/test-driver :h2
(with-transforms-enabled
(mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT ID, NAME FROM PUBLIC.PEOPLE")}
:target {:type "table" :schema "PUBLIC" :name "PEOPLE_SUMMARY" :database (mt/id)}}]
(let [run (fn [expected-status transform-test]
@@ -168,28 +172,75 @@
(testing "rejects a test that doesn't replace every table the transform reads"
(run 400 {:inputs [] :expectations []})))))))
+(defn- people-id-transform
+ "An MBQL transform selecting `ID` from `PEOPLE` into `PEOPLE_SUMMARY`."
+ []
+ (let [mp (mt/metadata-provider)]
+ {:source {:type :query
+ :query (-> (lib/query mp (lib.metadata/table mp (mt/id :people)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :people :id))]))}
+ :target {:type "table" :schema "PUBLIC" :name "PEOPLE_SUMMARY" :database (mt/id)}}))
+
(deftest permissions-test
(with-transforms-enabled
- (mt/with-temp [:model/Transform {transform-id :id} {}
- :model/TransformTest {test-id :id} {:transform_id transform-id}]
- (let [path (str "ee/transform-test/" test-id)
- body {:transform_id transform-id :name "nope" :inputs [] :expectations []}]
- (testing "users without transform access can't read or change transform tests"
- (mt/user-http-request :rasta :get 403 "ee/transform-test")
- (mt/user-http-request :rasta :get 403 path)
- (mt/user-http-request :rasta :post 403 "ee/transform-test" body)
- (mt/user-http-request :rasta :put 403 path {:name "nope"})
- (mt/user-http-request :rasta :delete 403 path)
- (mt/user-http-request :rasta :post 403 (str path "/run")))
- (testing "data analysts who can read but not write the transform can read but not change its tests"
+ (mt/with-temp [:model/Transform {transform-id :id} (people-id-transform)
+ :model/TransformTest {test-id :id} {:transform_id transform-id
+ :inputs inputs
+ :expectations expectations}]
+ (let [path (str "ee/transform-test/" test-id)
+ body {:transform_id transform-id :name "nope" :inputs inputs :expectations expectations}
+ refused-writes (fn [user]
+ (mt/user-http-request user :post 403 "ee/transform-test" body)
+ (mt/user-http-request user :put 403 path {:name "nope"})
+ (mt/user-http-request user :delete 403 path))]
+ (try
+ (testing "users without transform access can't read, change, or run transform tests"
+ (mt/user-http-request :rasta :get 403 "ee/transform-test")
+ (mt/user-http-request :rasta :get 403 path)
+ (mt/user-http-request :rasta :post 403 (str path "/run"))
+ (refused-writes :rasta))
(mt/with-data-analyst-role! (mt/user->id :lucky)
- (is (some #{test-id} (map :id (mt/user-http-request :lucky :get 200 "ee/transform-test"))))
- (mt/user-http-request :lucky :get 200 path)
- (mt/user-http-request :lucky :post 403 "ee/transform-test" body)
- (mt/user-http-request :lucky :put 403 path {:name "nope"})
- (mt/user-http-request :lucky :delete 403 path)
- (mt/user-http-request :lucky :post 403 (str path "/run"))))
- (is (t2/exists? :model/TransformTest :id test-id))))))
+ (testing "data analysts who can read transform can read and run its tests"
+ (mt/with-db-perms-for-group! (perms-group/all-users) (mt/id) {:perms/transforms :no
+ :perms/create-queries :query-builder}
+ (is (some #{test-id} (map :id (mt/user-http-request :lucky :get 200 "ee/transform-test"))))
+ (mt/user-http-request :lucky :get 200 path)
+ (is (= "passed" (:status (mt/user-http-request :lucky :post 200 (str path "/run")))))))
+ (testing "saving or deleting tests needs transforms permission on source database"
+ (mt/with-db-perms-for-group! (perms-group/all-users) (mt/id) {:perms/transforms :no
+ :perms/create-queries :query-builder-and-native}
+ (refused-writes :lucky)))
+ (testing "saving or deleting tests needs native access to source database"
+ (mt/with-db-perms-for-group! (perms-group/all-users) (mt/id) {:perms/transforms :yes
+ :perms/create-queries :query-builder}
+ (refused-writes :lucky)))
+ (testing "with both, data analysts can change tests"
+ (mt/with-db-perms-for-group! (perms-group/all-users) (mt/id) {:perms/transforms :yes
+ :perms/create-queries :query-builder-and-native}
+ (is (=? {:name "renamed"} (mt/user-http-request :lucky :put 200 path {:name "renamed"}))))))
+ (is (t2/exists? :model/TransformTest :id test-id))
+ (finally
+ (t2/delete! :model/TransformTestRun :initiated_by (mt/user->id :lucky))))))))
+
+(deftest update-transform-id-needs-access-to-both-transforms-test
+ (testing "moving test to another transform needs permission to save tests of both transforms"
+ (with-transforms-enabled
+ (mt/with-temp [:model/Database {other-db-id :id} {}
+ :model/Transform {transform-id :id} (people-id-transform)
+ :model/Transform {other-id :id} {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider other-db-id) "SELECT 1")}}
+ :model/TransformTest {test-id :id} {:transform_id transform-id
+ :inputs inputs
+ :expectations expectations}]
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (perms.tu/with-no-data-perms-for-all-users!
+ (mt/with-db-perms-for-group! (perms-group/all-users) (mt/id) {:perms/view-data :unrestricted
+ :perms/transforms :yes
+ :perms/create-queries :query-builder-and-native}
+ (let [path (str "ee/transform-test/" test-id)]
+ (mt/user-http-request :lucky :put 403 path {:transform_id other-id})
+ (is (= transform-id (t2/select-one-fn :transform_id :model/TransformTest :id test-id)))
+ (is (=? {:name "renamed"} (mt/user-http-request :lucky :put 200 path {:name "renamed"})))))))))))
(deftest list-runs-test
(with-transforms-enabled
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/expectations/report_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/expectations/report_test.clj
index f52c1f2aee1..666774ab037 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/expectations/report_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/expectations/report_test.clj
@@ -81,9 +81,8 @@
(is (= (vec (take expectations.report/row-cap rows)) kept)))))
(deftest capped-empty-test
- (is (= [[] 0] (expectations.report/capped [])))
- (is (= [[] 0] (expectations.report/capped nil))))
+ (is (= [[] 0] (expectations.report/capped []))))
(deftest capped-returns-a-vector-test
(testing "the sample is a vector, so a lazy seq cannot escape into the report"
- (is (vector? (first (expectations.report/capped (range 3)))))))
+ (is (vector? (first (expectations.report/capped (map vector (range 3))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
index 09f33a18a1a..80eb3eeb5ae 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
@@ -51,7 +51,8 @@
"A transform whose target name appears in none of the SQL below, so its own replacement entry
never interferes with an input's."
[]
- {:source {:type "query"}
+ {:source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table" :schema nil :name "UNREFERENCED_TARGET"}})
(defn- declared-input
@@ -281,8 +282,8 @@
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))
target "mb_remapping_probe_case_mismatch"]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ {:source {:type :query
:query (lib/native-query
mp (str "SELECT id, name FROM " schema "." (u/lower-case-en table)))}
:target {:type "table"
@@ -301,7 +302,7 @@
;; lower case. Refusing the run is a fine answer. Reporting :failed is not — it means
;; the fixture was ignored and the expectation was measured against real rows.
(let [outcome (try
- (transform-testing.runner/run-transform-test! transform-test)
+ (transform-testing.runner/run-saved-test! transform transform-test)
(catch Exception e
{:refused (ex-message e)}))]
(is (not= {:status :failed} outcome)
@@ -313,8 +314,8 @@
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))
target "mb_remapping_probe_target"]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." table))}
:target {:type "table"
:schema schema
@@ -328,5 +329,5 @@
:expectations [{:type :empty
:name "the output holds only the fixture row"
:sql (str "SELECT * FROM " schema "." target " WHERE id <> -1")}]}]
- (is (= :passed (:status (transform-testing.runner/run-transform-test! transform-test))))
+ (is (= :passed (:status (transform-testing.runner/run-saved-test! transform transform-test))))
(is (not (driver/table-exists? driver/*driver* (lib.metadata/database (mt/metadata-provider)) {:schema schema :name target}))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/reported_type_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/reported_type_test.clj
index a7d6be67d6b..0be199ecb08 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/reported_type_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/reported_type_test.clj
@@ -43,8 +43,8 @@
mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))
transform-sql (str "SELECT id AS total FROM " schema "." table)]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ {:source {:type :query
:query (lib/native-query mp transform-sql)}
:target {:type "table" :schema schema :name "people_total" :database (mt/id)}}
:model/TransformTest transform-test
@@ -55,7 +55,7 @@
:format :rows
:columns [{:name "total" :cast_type declared}]
:rows [{"total" 1}]}]}]
- (let [result (transform-testing.runner/run-transform-test! transform-test)]
+ (let [result (transform-testing.runner/run-saved-test! transform transform-test)]
(testing "the run passes, so nothing here reaches the author as a problem"
(is (= :passed (:status result))))
(testing "the transform's output column is not the type the author declared"
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
index 09990c9e289..93d58de2071 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
@@ -20,30 +20,30 @@
{:schema schema :name name}))
(defn- with-people-transform
- "Call `f` with the id of a transform selecting `id, name` from the people table, whose target is
+ "Call `f` with the schema and name of the people table and a transform selecting `id, name` from it into
`people_summary` in the same schema."
[f]
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." table))}
:target {:type "table"
:schema schema
:name "people_summary"
:database (mt/id)}}]
- (f schema table transform-id))))
+ (f schema table transform))))
(defn- run-test!
- "Run a transform test over `transform-id` with one SQL input standing in for `schema`.`table`."
- [schema table transform-id input-sql expectations]
+ "Run a transform test over `transform` with one SQL input standing in for `schema`.`table`."
+ [schema table transform input-sql expectations]
(mt/with-temp [:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table {:schema schema :name table}
:format :sql
:sql input-sql}]
:expectations expectations}]
- (transform-testing.runner/run-transform-test! transform-test)))
+ (transform-testing.runner/run-saved-test! transform transform-test)))
(defn- drivers-resolving-bare-tables
"The test drivers that resolve an unqualified table reference to the schema the test data lives in. Redshift reports
@@ -56,7 +56,7 @@
(deftest run-transform-test-rows-input-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(let [mp (mt/metadata-provider)
id-name (:name (lib.metadata/field mp (mt/id :people :id)))
name-name (:name (lib.metadata/field mp (mt/id :people :name)))
@@ -64,14 +64,14 @@
output (str schema ".people_summary")
run (fn [rows sql]
(mt/with-temp [:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table {:schema schema :name table}
:format :rows
:columns [{:name id-name :cast_type int-type}
{:name name-name :cast_type text-type}]
:rows rows}]
:expectations [{:type :empty :name "check" :sql sql}]}]
- (:status (transform-testing.runner/run-transform-test! transform-test))))
+ (:status (transform-testing.runner/run-saved-test! transform transform-test))))
two-rows [{id-name 1 name-name "abc"} {id-name 2 name-name "def"}]]
(testing "the transform reads the literal rows"
(is (= :passed (run two-rows (str "SELECT * FROM " output " WHERE id NOT IN (1, 2) OR name IS NULL"))))
@@ -87,8 +87,8 @@
{:name "name" :cast_type text-type}]))
(defn- check-empty-expectation!
- [schema table transform-id description where expected]
- (let [result (run-test! schema table transform-id one-row
+ [schema table transform description where expected]
+ (let [result (run-test! schema table transform one-row
[{:type :empty
:name description
:sql (str "SELECT * FROM " schema ".people_summary WHERE " where)}])]
@@ -99,18 +99,18 @@
(deftest run-transform-test-empty-expectation-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "passes when the query returns no rows"
- (check-empty-expectation! schema table transform-id "no rows" "name IS NULL" :passed))
+ (check-empty-expectation! schema table transform "no rows" "name IS NULL" :passed))
(testing "reads the input test data instead of the input table"
- (check-empty-expectation! schema table transform-id "input test data" "id <> 1" :passed))
+ (check-empty-expectation! schema table transform "input test data" "id <> 1" :passed))
(testing "fails when the query returns rows"
- (check-empty-expectation! schema table transform-id "rows returned" "name = 'abc'" :failed))))))
+ (check-empty-expectation! schema table transform "rows returned" "name = 'abc'" :failed))))))
(defn- check-persisted-run!
- [schema table transform-id where expected]
+ [schema table transform where expected]
(mt/with-temp [:model/TransformTest {transform-test-id :id :as transform-test}
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table {:schema schema :name table}
:format :sql
:sql one-row}]
@@ -118,7 +118,7 @@
:name "check"
:sql (str "SELECT * FROM " schema ".people_summary WHERE " where)}]}]
(binding [api/*current-user-id* (mt/user->id :crowberto)]
- (is (= expected (:status (transform-testing.runner/run-transform-test! transform-test)))))
+ (is (= expected (:status (transform-testing.runner/run-saved-test! transform transform-test)))))
(let [run (t2/select-one :model/TransformTestRun :transform_test_id transform-test-id)]
(is (= expected (:status run)))
(is (= (mt/user->id :crowberto) (:initiated_by run)))
@@ -128,27 +128,27 @@
(deftest run-transform-test-persists-passed-run-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
- (check-persisted-run! schema table transform-id "name IS NULL" :passed)))))
+ (fn [schema table transform]
+ (check-persisted-run! schema table transform "name IS NULL" :passed)))))
(deftest run-transform-test-persists-failed-run-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
- (check-persisted-run! schema table transform-id "id = 1" :failed)))))
+ (fn [schema table transform]
+ (check-persisted-run! schema table transform "id = 1" :failed)))))
(deftest run-transform-test-persists-post-validation-error-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(mt/with-temp [:model/TransformTest {transform-test-id :id :as transform-test}
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table {:schema schema :name table}
:format :sql
:sql "SELECT * FROM transform_test_input_that_does_not_exist"}]
:expectations []}]
(is (thrown? ExceptionInfo
- (transform-testing.runner/run-transform-test! transform-test)))
+ (transform-testing.runner/run-saved-test! transform transform-test)))
(let [run (t2/select-one :model/TransformTestRun :transform_test_id transform-test-id)]
(is (= :error (:status run)))
(is (some? (:end_time run)))))))))
@@ -156,9 +156,9 @@
(deftest run-transform-test-equals-passes-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "passes when the output matches the expected rows"
- (let [result (run-test! schema table transform-id one-row
+ (let [result (run-test! schema table transform one-row
[{:type :equals :name "output" :format :rows
:columns (id+name)
:rows [{"id" 1 "name" "abc"}]}])]
@@ -168,9 +168,9 @@
(deftest run-transform-test-equals-cell-mismatch-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "a single differing cell is reported by column name"
- (let [result (run-test! schema table transform-id one-row
+ (let [result (run-test! schema table transform one-row
[{:type :equals :name "output" :format :rows
:columns (id+name)
:rows [{"id" 1 "name" "xyz"}]}])
@@ -184,9 +184,9 @@
(deftest run-transform-test-equals-ignores-undeclared-columns-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "a column the expectation does not declare is not compared"
- (let [result (run-test! schema table transform-id one-row
+ (let [result (run-test! schema table transform one-row
[{:type :equals :name "id only" :format :rows
:columns [{:name "id" :cast_type (first (transform-testing.test-util/cast-types))}]
:rows [{"id" 1}]}])]
@@ -195,10 +195,10 @@
(deftest run-transform-test-equals-is-multiset-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "a duplicated output row fails against a single expected row"
;; Set difference (a plain EXCEPT) passes here; multiset difference must not.
- (let [result (run-test! schema table transform-id
+ (let [result (run-test! schema table transform
(str "SELECT id, name FROM (" one-row " UNION ALL " one-row ") AS t")
[{:type :equals :name "output" :format :rows
:columns (id+name)
@@ -212,11 +212,11 @@
(deftest run-transform-test-equals-matches-null-cells-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
- (fn [schema table transform-id]
+ (fn [schema table transform]
(testing "a NULL cell in the output cancels against a nil in the expectation"
;; `GROUP BY` folds NULLs together. An equality-based difference leaves the row
;; uncancelled, reporting it as both missing and extra.
- (let [result (run-test! schema table transform-id
+ (let [result (run-test! schema table transform
(str "SELECT 1 AS id, CAST(NULL AS " (second (transform-testing.test-util/cast-types)) ") AS name")
[{:type :equals :name "output" :format :rows
:columns (id+name)
@@ -230,15 +230,15 @@
;; people pointing at the real table (read production, false-green). Must reject before running.
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." table))}
:target {:type "table" :schema schema :name "people_summary" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs []
:expectations [{:type :empty :name "all" :sql (str "SELECT * FROM " schema ".people_summary")}]}]
- (let [ex (try (transform-testing.runner/run-transform-test! transform-test)
+ (let [ex (try (transform-testing.runner/run-saved-test! transform transform-test)
nil
(catch ExceptionInfo e e))]
(testing "throws"
@@ -257,19 +257,19 @@
(let [mp (mt/metadata-provider)
{pschema :schema, people :name} (table-ref :people)
{oschema :schema, orders :name} (table-ref :orders)]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query
mp (str "SELECT o.id, p.name"
" FROM " oschema "." orders " o"
" JOIN " pschema "." people " p ON o.user_id = p.id"))}
:target {:type "table" :schema oschema :name "order_owners" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table {:schema oschema :name orders} :format :sql
:sql "SELECT 1 AS id, 1 AS user_id"}]
:expectations [{:type :empty :name "all" :sql (str "SELECT * FROM " oschema ".order_owners")}]}]
- (let [ex (try (transform-testing.runner/run-transform-test! transform-test)
+ (let [ex (try (transform-testing.runner/run-saved-test! transform transform-test)
nil
(catch ExceptionInfo e e))]
(is (= ::transform-testing.errors/missing-inputs (:error-type (ex-data ex))))
@@ -284,18 +284,18 @@
(let [mp (mt/metadata-provider)
{schema :schema, people :name} (lib.metadata/table mp (mt/id :people))
orders (:name (lib.metadata/table mp (mt/id :orders)))]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." people))}
:target {:type "table" :schema schema :name "people_summary" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table (table-ref :people) :format :sql
:sql "SELECT 1 AS id, 'x' AS name"}
{:table (table-ref :orders) :format :sql
:sql "SELECT 1 AS id"}]
:expectations [{:type :empty :name "all" :sql (str "SELECT * FROM " schema ".people_summary")}]}]
- (let [ex (try (transform-testing.runner/run-transform-test! transform-test)
+ (let [ex (try (transform-testing.runner/run-saved-test! transform transform-test)
nil
(catch ExceptionInfo e e))]
(is (= ::transform-testing.errors/unused-inputs (:error-type (ex-data ex))))
@@ -310,16 +310,16 @@
;; and silently drop a fixture. Reject on :table, not on the whole (distinct) input map.
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT id, name FROM " schema "." table))}
:target {:type "table" :schema schema :name "people_summary" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table (table-ref :people) :format :sql :sql "SELECT 1 AS id, 'a' AS name"}
{:table (table-ref :people) :format :sql :sql "SELECT 2 AS id, 'b' AS name"}]
:expectations [{:type :empty :name "all" :sql (str "SELECT * FROM " schema ".people_summary")}]}]
- (let [ex (try (transform-testing.runner/run-transform-test! transform-test)
+ (let [ex (try (transform-testing.runner/run-saved-test! transform transform-test)
nil
(catch ExceptionInfo e e))]
(is (= ::transform-testing.errors/duplicate-input-table (:error-type (ex-data ex))))
@@ -330,13 +330,13 @@
(binding [sql-tools.settings/*parser-backend-override* :sqlglot]
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(let [mp (mt/metadata-provider)]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp "SELECT * FROM WHERE (")}
:target {:type "table" :schema "public" :name "parse_test" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id :inputs [] :expectations []}]
- (let [e (try (transform-testing.runner/run-transform-test! transform-test)
+ {:transform_id (:id transform) :inputs [] :expectations []}]
+ (let [e (try (transform-testing.runner/run-saved-test! transform transform-test)
(catch ExceptionInfo e e))]
(is (= ::transform-testing.errors/unparseable-source (:error-type (ex-data e))))
(testing "and the refusal reads as the parser's own diagnostic, less its internals"
@@ -351,15 +351,15 @@
(mt/test-drivers (drivers-resolving-bare-tables)
(let [mp (mt/metadata-provider)
{schema :schema, table :name} (lib.metadata/table mp (mt/id :people))]
- (mt/with-temp [:model/Transform {transform-id :id}
- {:source {:type "query"
+ (mt/with-temp [:model/Transform transform
+ {:source {:type :query
:query (lib/native-query mp (str "SELECT " table ".id, " table ".name FROM " table))}
:target {:type "table" :schema schema :name "people_qualified" :database (mt/id)}}
:model/TransformTest transform-test
- {:transform_id transform-id
+ {:transform_id (:id transform)
:inputs [{:table (table-ref :people) :format :sql
:sql "SELECT 1 AS id, 'x' AS name"}]
:expectations [{:type :empty :name "only the input row"
:sql (str "SELECT * FROM " schema ".people_qualified WHERE id <> 1")}]}]
(is (=? {:status :passed}
- (transform-testing.runner/run-transform-test! transform-test))))))))
+ (transform-testing.runner/run-saved-test! transform transform-test))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index 97980cafe2b..eb6fc334166 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -20,7 +20,7 @@
(defn- query-transform-payload
[table-name]
{:name "Test Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:schema (get-test-schema)
@@ -29,7 +29,7 @@
(defn- python-transform-map
[table-name]
{:name "Python Transform"
- :source {:type "python"
+ :source {:type :python
:body "print('hello world')"
:source-tables []
:source-database (mt/id)}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
index 13a059e68f4..3f3508a93d3 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
@@ -43,14 +43,16 @@
(with-redefs [t/offset-date-time (constantly frozen-today)]
(mt/with-model-cleanup [:model/TransformRun]
(mt/with-temp [:model/Transform {mbql-id :id}
- {:source {:type "query"
+ {:source {:type :query
:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
:target {:type "table"
:name (str "test_mbql_" (random-uuid))
:database (mt/id)}}
:model/Transform {python-id :id}
- {:source {:type "python"
- :source-database (mt/id)}
+ {:source {:type :python
+ :source-database (mt/id)
+ :source-tables []
+ :body "print('hello world')"}
:target {:type "table"
:name (str "test_python_" (random-uuid))
:database (mt/id)}}]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 3825be149ec..de957075009 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -77,6 +77,18 @@
(transforms.execute/execute! transform run-opts))
(transforms.execute/execute! transform run-opts)))
+(defn- native-query-with-table-tag
+ "Create a native query with a `source_table` table tag."
+ [sql table-id]
+ (lib/with-template-tags
+ (lib/native-query (mt/metadata-provider) sql)
+ {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type :table
+ :table-id table-id
+ :required true}}))
+
(defn- make-incremental-source-query-with-table-tag
"Create a native query with a table tag for testing subquery expansion."
[checkpoint-config]
@@ -88,25 +100,21 @@
(quoted "name") (quoted "category") (quoted "price") (quoted "created_at")
timestamp-sql (quoted "load_timestamp") (quoted "source_table")
(quoted field-name))]
- {:database (mt/id)
- :type :native
- :native {:query query
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}))
+ (native-query-with-table-tag query table-id)))
(defn- make-incremental-mbql-query
"Create an MBQL query for incremental transforms. "
[checkpoint-config]
- (let [{:keys [field-name]} checkpoint-config]
- (mt/mbql-query transforms_products
- {:expressions {"load_timestamp" [:now]}
- :fields [$name $category $price $created_at [:expression "load_timestamp"]]
- :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]})))
+ (let [{:keys [field-name]} checkpoint-config
+ mp (mt/metadata-provider)
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :transforms_products)))
+ (lib/expression "load_timestamp" (lib/now)))
+ col #(lib.metadata/field mp (mt/id :transforms_products %))]
+ (-> query
+ (lib/with-fields [(col :name) (col :category) (col :price) (col :created_at)
+ (lib/expression-ref query "load_timestamp")])
+ (lib/order-by (col (keyword field-name))))))
(def incremental-python-body
(str "import pandas as pd\n"
@@ -128,15 +136,15 @@
:source_database_id (mt/id)
:source (case transform-type
;; Native queries now use table tags with checkpoint-filter-field-id
- :native {:type "query"
+ :native {:type :query
:query (make-incremental-source-query-with-table-tag checkpoint-config)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id (checkpoint-filter-field)}}
- :mbql {:type "query"
+ :mbql {:type :query
:query (make-incremental-mbql-query checkpoint-config)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id (checkpoint-filter-field)}}
- :python {:type "python"
+ :python {:type :python
:source-tables [(transforms.tu/source-table-entry "transforms_products" (mt/id :transforms_products))]
:body incremental-python-body
:source-incremental-strategy {:type "checkpoint"
@@ -157,11 +165,10 @@
(defn- get-distinct-timestamp-count
"Get the count of distinct load_timestamp values in a table."
[{table-name :name, :keys [schema database]}]
- (let [native-query {:database database
- :type :native
- :native {:query (format "SELECT COUNT(DISTINCT %s) FROM %s"
- (sql.u/quote-name driver/*driver* :field "load_timestamp")
- (sql.u/quote-name driver/*driver* :table schema table-name))}}
+ (let [native-query (lib/native-query (lib-be/application-database-metadata-provider database)
+ (format "SELECT COUNT(DISTINCT %s) FROM %s"
+ (sql.u/quote-name driver/*driver* :field "load_timestamp")
+ (sql.u/quote-name driver/*driver* :table schema table-name)))
result (qp/process-query native-query)]
(some-> result :data :rows first first bigint)))
@@ -386,7 +393,7 @@
(testing "Switch to non-incremental via PUT API"
(let [non-incremental-payload (-> initial-payload
(update :source dissoc :source-incremental-strategy)
- (update :target dissoc :source-incremental-strategy)
+ (update :target dissoc :target-incremental-strategy)
(update :target assoc :type "table"))
updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))
non-incremental-payload)]
@@ -421,7 +428,7 @@
incremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)
initial-payload (-> incremental-payload
(update :source dissoc :source-incremental-strategy)
- (update :target dissoc :source-incremental-strategy)
+ (update :target dissoc :target-incremental-strategy)
(update :target assoc :type "table"))]
(mt/with-temp [:model/Transform transform initial-payload]
(testing "Initial non-incremental run creates table"
@@ -481,16 +488,8 @@
table-id (mt/id :transforms_products)
schema (t2/select-one-fn :schema :model/Table :id table-id)
transform-payload {:name "Invalid Checkpoint Type Transform"
- :source {:type "query"
- :query {:database (mt/id)
- :type :native
- :native {:query "SELECT * FROM {{source_table}} AS s"
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}
+ :source {:type :query
+ :query (native-query-with-table-tag "SELECT * FROM {{source_table}} AS s" table-id)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id name-field-id}}
:target {:type "table-incremental"
@@ -516,16 +515,8 @@
table-id (mt/id :transforms_products)
checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)
;; Query that returns no rows but has the checkpoint column
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT * FROM {{source_table}} AS s WHERE 1 = 2"
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}
+ source {:type :query
+ :query (native-query-with-table-tag "SELECT * FROM {{source_table}} AS s WHERE 1 = 2" table-id)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id checkpoint-field-id}}
target {:type "table-incremental"
@@ -553,16 +544,8 @@
(let [db-id (mt/id)
table-id (mt/id :transforms_products)
invalid-field-id 999999
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT id FROM {{source_table}}"
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}
+ source {:type :query
+ :query (native-query-with-table-tag "SELECT id FROM {{source_table}}" table-id)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id invalid-field-id}}
target {:type "table-incremental"
@@ -586,16 +569,8 @@
(let [db-id (mt/id)
table-id (mt/id :transforms_products)
field-id (mt/id :transforms_products :id)
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT id FROM {{source_table}} as s"
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}
+ source {:type :query
+ :query (native-query-with-table-tag "SELECT id FROM {{source_table}} as s" table-id)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id field-id}}
target {:type "table-incremental"
@@ -626,16 +601,8 @@
table-id (mt/id :transforms_products)
checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)
make-source (fn [query-sql]
- {:type "query"
- :query {:database db-id
- :type :native
- :native {:query query-sql
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}}
+ {:type :query
+ :query (native-query-with-table-tag query-sql table-id)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id checkpoint-field-id}})
source (make-source "SELECT id FROM {{source_table}} AS s")
@@ -678,7 +645,7 @@
{:keys [field-name expected-initial-checkpoint]} checkpoint-config
checkpoint-filter-field-id (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products))
transform-payload {:name "Native With Table Tag"
- :source {:type "query"
+ :source {:type :query
:query (make-incremental-source-query-with-table-tag checkpoint-config)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id checkpoint-filter-field-id}}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
index 6b0a677d57d..56d4603eabe 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
@@ -7,6 +7,7 @@
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql.util :as sql.u]
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
[metabase.test :as mt]
@@ -83,25 +84,31 @@
:schema (t2/select-one-fn :schema :model/Table :id (mt/id :order_status))
:database (mt/id)})
+(defn- native-query-with-table-tag
+ "Create a native query whose `tag-name` template tag points at `table-id`."
+ [sql tag-name display-name table-id]
+ (lib/with-template-tags
+ (lib/native-query (mt/metadata-provider) sql)
+ {tag-name {:id tag-name
+ :name tag-name
+ :display-name display-name
+ :type :table
+ :table-id table-id
+ :required true}}))
+
(defn- merge-source-query []
- {:database (mt/id)
- :type :native
- :native {:query (format "SELECT %s, %s FROM {{order_status}} AS %s"
- (sql.u/quote-name driver/*driver* :field "order_id")
- (sql.u/quote-name driver/*driver* :field "status")
- (sql.u/quote-name driver/*driver* :field "order_status"))
- :template-tags {"order_status" {:id "order_status"
- :name "order_status"
- :display-name "Order Status"
- :type "table"
- :table-id (mt/id :order_status)
- :required true}}}})
+ (native-query-with-table-tag
+ (format "SELECT %s, %s FROM {{order_status}} AS %s"
+ (sql.u/quote-name driver/*driver* :field "order_id")
+ (sql.u/quote-name driver/*driver* :field "status")
+ (sql.u/quote-name driver/*driver* :field "order_status"))
+ "order_status" "Order Status" (mt/id :order_status)))
(defn- merge-transform-payload [target-table & {:keys [validate?] :or {validate? true}}]
(let [seq-field-id (t2/select-one-pk :model/Field :name "change_seq" :table_id (mt/id :order_status))]
{:name "Merge Transform"
:source_database_id (mt/id)
- :source {:type "query"
+ :source {:type :query
:query (merge-source-query)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id seq-field-id}}
@@ -169,19 +176,13 @@
(let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
ts-field-id (t2/select-one-pk :model/Field :name "changed_at"
:table_id (mt/id :order_status_ts))
- source-query {:database (mt/id)
- :type :native
- :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"
- (q "order_id") (q "status") (q "t"))
- :template-tags {"t" {:id "t"
- :name "t"
- :display-name "T"
- :type "table"
- :table-id (mt/id :order_status_ts)
- :required true}}}}
+ source-query (native-query-with-table-tag
+ (format "SELECT %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "status") (q "t"))
+ "t" "T" (mt/id :order_status_ts))
payload {:name "Temporal Merge"
:source_database_id (mt/id)
- :source {:type "query"
+ :source {:type :query
:query source-query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id}}
@@ -256,19 +257,13 @@
(let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
ts-field-id (t2/select-one-pk :model/Field :name "changed_at"
:table_id (mt/id :order_status_lb))
- source-query {:database (mt/id)
- :type :native
- :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"
- (q "order_id") (q "status") (q "t"))
- :template-tags {"t" {:id "t"
- :name "t"
- :display-name "T"
- :type "table"
- :table-id (mt/id :order_status_lb)
- :required true}}}}
+ source-query (native-query-with-table-tag
+ (format "SELECT %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "status") (q "t"))
+ "t" "T" (mt/id :order_status_lb))
payload {:name "Lookback Merge"
:source_database_id (mt/id)
- :source {:type "query"
+ :source {:type :query
:query source-query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id
@@ -370,10 +365,10 @@
:effective_type :type/Instant})
payload {:name "Unix Timestamp Lookback Merge"
:source_database_id (mt/id)
- :source {:type "query"
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :order_status_unix)}}
+ :source {:type :query
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider)
+ (mt/id :order_status_unix)))
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id
:lookback {:value 1 :unit "day"}}}
@@ -435,19 +430,13 @@
(let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
seq-field-id (t2/select-one-pk :model/Field :name "change_seq"
:table_id (mt/id :order_region_status))
- source-query {:database (mt/id)
- :type :native
- :native {:query (format "SELECT %s, %s, %s FROM {{t}} AS %s"
- (q "order_id") (q "region") (q "status") (q "t"))
- :template-tags {"t" {:id "t"
- :name "t"
- :display-name "T"
- :type "table"
- :table-id (mt/id :order_region_status)
- :required true}}}}
+ source-query (native-query-with-table-tag
+ (format "SELECT %s, %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "region") (q "status") (q "t"))
+ "t" "T" (mt/id :order_region_status))
payload {:name "Composite Merge"
:source_database_id (mt/id)
- :source {:type "query"
+ :source {:type :query
:query source-query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id seq-field-id}}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index 48d8539f124..c8226df63f8 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -12,7 +12,7 @@
[source-tables target-name & [target-schema]]
(let [default-schema "public"
schema (or target-schema default-schema)]
- {:source {:type "python"
+ {:source {:type :python
:source-database (mt/id)
:source-tables source-tables
:body "df.write_output()"}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/search_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/search_test.clj
index 6e2415497a0..60df4d73363 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/search_test.clj
@@ -20,10 +20,11 @@
(let [now (t/truncate-to (t/offset-date-time) :millis)]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Test Python transform"
:description "A Python test transform"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
+ :source-tables []
:body "import pandas as pd\n"}
- :target {:database (mt/id)}
+ :target {:type "table" :database (mt/id) :name "test_table"}
:created_at now
:updated_at now}]
(let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test Python transform")]
@@ -39,9 +40,10 @@
(deftest transform-query-ingestion-test
(testing "Contents of Python transform sources are extracted and indexed for full-text search"
(when (= (mdb/db-type) :postgres)
- (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
- :source {:type "python"
+ (mt/with-temp [:model/Transform _ {:target {:type "table" :database (mt/id) :name "python_target"}
+ :source {:type :python
:source-database (mt/id)
+ :source-tables []
:body "import pandas as pd\n"}
:name "Test python transform"}]
(let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test python transform")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
index d43032951c2..e7fe0068313 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
@@ -19,5 +19,5 @@
(is (transforms.u/is-temp-transform-table? table-with-schema)))))
(testing "Ignores non-transform tables"
(mt/with-premium-features #{:transforms-basic}
- (is (false? (transforms.u/is-temp-transform-table? {:name :orders})))
- (is (false? (transforms.u/is-temp-transform-table? {:name :public/orders}))))))
+ (is (false? (transforms.u/is-temp-transform-table? {:name "orders"})))
+ (is (false? (transforms.u/is-temp-transform-table? {:name "public/orders"}))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj
index 3f596c390d8..a3d5192aec7 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj
@@ -122,11 +122,7 @@
(defn- native-orders-query
"Build a native-only MBQL 5 query selecting from the orders table."
[]
- {:lib/type :mbql/query
- :lib/metadata (mt/metadata-provider)
- :database (mt/id)
- :stages [{:lib/type :mbql.stage/native
- :native "SELECT * FROM ORDERS"}]})
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM ORDERS"))
(deftest extract-sources-mbql-test
(testing "MBQL: extracts source table from simple query"
@@ -163,11 +159,8 @@
(deftest extract-sources-native-with-joins-test
(testing "Native: extracts multiple source tables from SQL with JOIN"
(let [transform {:source {:type :query
- :query {:lib/type :mbql/query
- :lib/metadata (mt/metadata-provider)
- :database (mt/id)
- :stages [{:lib/type :mbql.stage/native
- :native "SELECT * FROM ORDERS JOIN PRODUCTS ON ORDERS.PRODUCT_ID = PRODUCTS.ID"}]}}
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM ORDERS JOIN PRODUCTS ON ORDERS.PRODUCT_ID = PRODUCTS.ID")}
:name "test"}
sources (context/extract-sources transform)
table-ids (set (map :table-id sources))]
@@ -179,7 +172,8 @@
(testing "Python: extracts source tables from source-tables vec"
(let [transform {:source {:type :python
:source-tables [{:alias "orders" :table_id (mt/id :orders)}]
- :source-database (mt/id)}
+ :source-database (mt/id)
+ :body "pass"}
:name "test"}
sources (context/extract-sources transform)]
(is (seq sources))
@@ -191,7 +185,8 @@
(let [transform {:source {:type :python
:source-tables [{:alias "orders" :table_id (mt/id :orders)}
{:alias "products" :table_id (mt/id :products)}]
- :source-database (mt/id)}
+ :source-database (mt/id)
+ :body "pass"}
:name "test"}
sources (context/extract-sources transform)
table-ids (set (map :table-id sources))]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_inspector_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_inspector_test.clj
index 327aeb12ac5..ae365216ef8 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_inspector_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_inspector_test.clj
@@ -3,6 +3,8 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-inspector.core :as inspector]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.test-util :as transforms.tu]
@@ -13,6 +15,21 @@
(defn- default-schema []
(t2/select-one-fn :schema :model/Table :id (mt/id :orders)))
+(defn- orders-query
+ "An MBQL query reading the `orders` table."
+ []
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id :orders)))))
+
+(defn- orders-products-query
+ "An MBQL query reading the `orders` table left-joined to `products`."
+ []
+ (let [mp (mt/metadata-provider)]
+ (lib/join (orders-query)
+ (lib/join-clause (lib.metadata/table mp (mt/id :products))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))
+ (lib.metadata/field mp (mt/id :products :id)))]))))
+
(defn- make-mbql-transform
"Create a transform map for testing with an MBQL query."
[query target-name & [schema]]
@@ -21,7 +38,7 @@
:name (str "inspector_test_" target-name)
:target {:schema (or schema (default-schema))
:name target-name
- :type :table}})
+ :type "table"}})
;; TODO: these tests create new tables in the default schema, which is fine for
;; most drivers, but not for cloud DBs like snowflake and bigquery, because
@@ -43,9 +60,7 @@
:schema (default-schema)
:name "g_inspector_nr"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ (orders-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform transform transform-data]
@@ -64,9 +79,7 @@
:schema (default-schema)
:name "g_inspector_rdy"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ (orders-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
@@ -92,9 +105,7 @@
:schema (default-schema)
:name "g_inspector_src"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ (orders-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
@@ -126,9 +137,7 @@
:schema (default-schema)
:name "g_inspector_gs"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ (orders-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
@@ -156,19 +165,7 @@
:schema (default-schema)
:name "g_inspector_ja"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)
- :joins [{:fields "all"
- :strategy "left-join"
- :alias "Products"
- :condition [:=
- [:field (mt/id :orders :product_id)
- {:base-type "type/Integer"}]
- [:field (mt/id :products :id)
- {:base-type "type/Integer"
- :join-alias "Products"}]]
- :source-table (mt/id :products)}]}}
+ (orders-products-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
@@ -196,9 +193,7 @@
:schema (default-schema)
:name "g_inspector_na"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ (orders-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
@@ -220,19 +215,7 @@
:schema (default-schema)
:name "g_inspector_ji"}]
(let [transform-data (make-mbql-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)
- :joins [{:fields "all"
- :strategy "left-join"
- :alias "Products"
- :condition [:=
- [:field (mt/id :orders :product_id)
- {:base-type "type/Integer"}]
- [:field (mt/id :products :id)
- {:base-type "type/Integer"
- :join-alias "Products"}]]
- :source-table (mt/id :products)}]}}
+ (orders-products-query)
target-name
(default-schema))]
(mt/with-temp [:model/Transform {tid :id :as transform} transform-data]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index 9cc88a7d2d9..3eda7032f4e 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -32,7 +32,7 @@
:database (mt/id)}
transform-def {:name (str "E2E Transform Test " table-name)
:source_database_id (mt/id)
- :source {:type "python"
+ :source {:type :python
:source-tables source-tables
:body transform-code}
:target target}]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
index 9020507dc3d..a41f18c6c93 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
@@ -43,7 +43,7 @@
" }\n"
" return pd.DataFrame(data)")
transform-payload {:name "Library Test Transform"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables []
:body transform-body}
@@ -69,7 +69,7 @@
:schema schema
:name "target_table"}]
(let [original {:name "Gadget Products"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index 77fadef1809..a853bcafa9a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -27,7 +27,7 @@
:schema schema
:name "swap_tbl"}]
(let [initial-transform {:name "Python Transform Initial"
- :source {:type "python"
+ :source {:type :python
:source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
@@ -41,7 +41,7 @@
(let [initial-rows (transforms.tu/table-rows table-name)]
(is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
(t2/update! :model/Transform :id (:id transform)
- {:source {:type "python"
+ {:source {:type :python
:source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
@@ -76,7 +76,7 @@
:schema schema
:name "cleanup_"}]
(let [transform-def {:name "Python Transform Cleanup"
- :source {:type "python"
+ :source {:type :python
:source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
@@ -112,7 +112,7 @@
" time.sleep(10) # Sleep longer than timeout\n"
" return pd.DataFrame({'result': ['should_not_reach_here']})")
transform-def {:name "Python Transform Timeout Test"
- :source {:type "python"
+ :source {:type :python
:source-tables []
:source-database (mt/id)
:body long-running-code}
@@ -140,7 +140,7 @@
:schema schema
:name "unresolved_test"}]
(mt/with-temp [:model/Transform transform {:name "Transform with unresolved ref"
- :source {:type "python"
+ :source {:type :python
:body "def transform(input): return input"
:source-database (mt/id)
:source-tables [{:alias "input"
@@ -163,7 +163,7 @@
:schema nil
:name "unresolved_no_schema_test"}]
(mt/with-temp [:model/Transform transform {:name "Transform with unresolved ref no schema"
- :source {:type "python"
+ :source {:type :python
:body "def transform(input): return input"
:source-database (mt/id)
:source-tables [{:alias "input"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
index c4dbe7a57f5..2fe5492a9e8 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -26,7 +26,7 @@
(mt/with-temp
[:model/TransformTag {tag-id :id} {:name "every second"}
:model/Transform {transform-id :id} {:name "Gadget Products"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
@@ -48,27 +48,33 @@
:timeout-ms 10000
:interval-ms 1000})))))))))))
-(def ^:private python-source {:type "python"})
+(def ^:private python-source {:type :python :source-tables [] :body "pass"})
(deftest run-transform-feature-flag-test
(testing "Python transforms are skipped without :transforms-python feature"
(mt/with-premium-features #{:transforms-basic}
- (let [python-transform {:id 2
- :source python-source
- :name "Test Python Transform"}
- run-id 101]
- (mt/with-log-messages-for-level [messages [metabase.transforms.jobs :warn]]
- (mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)]
- (#'jobs/run-transform! {:parent-run [:job run-id]
- :run-method :scheduled
- :user-id nil
- :add-run-activity! (constantly nil)}
- (promise)
- python-transform)
- (is (= 1 (count (messages)))
- "Should log exactly one warning")
- (is (= :warn (:level (first (messages))))
- "Should log at warn level")
- (is (re-matches #".*Skip running transform 2 due to lacking premium features.*"
- (:message (first (messages))))
- "Warning message should indicate transform was skipped due to missing features")))))))
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "Test Python Transform"
+ :source python-source
+ :target {:type "table" :name "python_feature_flag_target"}}]
+ (let [python-transform (t2/select-one :model/Transform :id transform-id)
+ run-id 101]
+ (mt/with-log-messages-for-level [messages [metabase.transforms.jobs :warn]]
+ (mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)]
+ (#'jobs/run-transform! {:plan []
+ :deps {}
+ :parent-run [:job run-id]
+ :run-gone? (constantly false)
+ :run-method :cron
+ :user-id nil
+ :add-run-activity! (constantly nil)
+ :lanes {:sql 1 :py 1}
+ :timeout-ms 1000}
+ (promise)
+ python-transform)
+ (is (= 1 (count (messages)))
+ "Should log exactly one warning")
+ (is (= :warn (:level (first (messages))))
+ "Should log at warn level")
+ (is (re-matches (re-pattern (str ".*Skip running transform " transform-id " due to lacking premium features.*"))
+ (:message (first (messages))))
+ "Warning message should indicate transform was skipped due to missing features"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index aca3394d5c4..fd120c544a1 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -1,6 +1,8 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.ordering-test
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as ordering]
@@ -21,7 +23,7 @@
(let [name (or name (mt/random-name))
schema (or schema (transforms.tu/default-schema-or-public))
target-db (or target-db (mt/id))]
- {:source {:type "python"
+ {:source {:type :python
:source-tables source-tables
:body "# Python code here\nresult = df"}
:name (str "python_transform_" name)
@@ -85,16 +87,13 @@
:model/Table {table2 :id} {:schema schema :name "python_output"}
:model/Field _ {:table_id table2 :name "bar"}
;; SQL transform that depends on orders table
- :model/Transform {t1 :id} (sql-tx {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}
+ :model/Transform {t1 :id} (sql-tx (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))
"sql_output")
;; Python transform that depends on the SQL transform's output
:model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "sql_output" table1)] "python_output")
;; Another SQL transform that depends on the Python transform's output
- :model/Transform {t3 :id} (sql-tx {:database (mt/id)
- :type "query"
- :query {:source-table table2}} "final_output")]
+ :model/Transform {t3 :id} (sql-tx (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) table2))
+ "final_output")]
(t2/update! :model/Transform :id t1 {:target_table_id table1})
(t2/update! :model/Transform :id t2 {:target_table_id table2})
(is (= {t1 #{}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index a3a703f8027..462a64b852a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -31,7 +31,7 @@
(letfn [(create-transform! []
(let [schema (get-test-schema)
transform-payload {:name "My beautiful python runner"
- :source {:type "python"
+ :source {:type :python
:body "print('hello world')"
:source-tables []
:source-database (mt/id)}
@@ -47,7 +47,7 @@
(is (= "Premium features required for this transform type are not enabled."
(mt/user-http-request :lucky :post 402 "transform"
{:name "My beautiful python runner"
- :source {:type "python"
+ :source {:type :python
:body "print('hello world')"
:source-tables []
:source-database (mt/id)}
@@ -61,7 +61,7 @@
(is (= "Premium features required for this transform type are not enabled."
(mt/user-http-request :lucky :post 402 "transform"
{:name "My beautiful python runner"
- :source {:type "python"
+ :source {:type :python
:body "print('hello world')"
:source-tables []
:source-database (mt/id)}
@@ -76,7 +76,7 @@
(is (= "print('hello chris')"
(-> (mt/user-http-request :lucky :put 200 (format "transform/%s" (:id transform))
{:name "My beautiful python runner"
- :source {:type "python"
+ :source {:type :python
:body "print('hello chris')"
:source-tables []
:source-database (mt/id)}
@@ -92,7 +92,7 @@
(testing "Updating a python transform requires both :transforms-basic and :transforms-python features"
(mt/with-temp [:model/Transform {id :id
:as transform} {:name "Original Python Transform"
- :source {:type "python"
+ :source {:type :python
:body "print('original')"
:source-tables []
:source-database (mt/id)}
@@ -116,7 +116,7 @@
(with-transform-cleanup! [table-name "test_run_python"]
(let [schema (get-test-schema)
transform-payload {:name "Test Run Python Transform"
- :source {:type "python"
+ :source {:type :python
:body "def transform():\n pass"
:source-tables []
:source-database (mt/id)}
@@ -145,7 +145,7 @@
:schema schema
:name "target_table"}]
(let [original {:name "Gadget Products"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
@@ -223,7 +223,7 @@
{:post [(integer? %)]}
(:id (mt/user-http-request :crowberto :post 200 "transform"
{:name "Python logging test"
- :source {:type "python"
+ :source {:type :python
:body (program->source program)
:source-database (mt/id)
:source-tables [(transforms.tu/default-source-table-entry)]}
@@ -328,7 +328,7 @@
:name "python_target_table"
:active true}
:model/Transform transform {:name "Python Transform Cross DB"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/default-source-table-entry)]
:body "def transform():\n pass"}
@@ -351,7 +351,7 @@
(letfn [(create-transform [{:keys [desc program]} target]
(mt/user-http-request :crowberto :post 200 "transform"
{:name (format "Python cancellation test: %s" desc)
- :source {:type "python"
+ :source {:type :python
:body (str/join "\n" program)
:source-database (mt/id)
:source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]}
@@ -515,7 +515,7 @@
:schema schema
:name "schema_change_test"}]
(let [initial-transform {:name "Schema Change Integration Test"
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/default-source-table-entry)]
:body (str "import pandas as pd\n"
@@ -532,7 +532,7 @@
(is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob with ages"))
;; Update transform with different schema via API endpoint
(let [updated-transform (assoc initial-transform
- :source {:type "python"
+ :source {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/default-source-table-entry)]
:body (str "import pandas as pd\n"
@@ -569,7 +569,7 @@
:schema (get-test-schema)
:table "transforms_products"}]
transform-payload {:name "Transform with table ref"
- :source {:type "python"
+ :source {:type :python
:body "def transform(input):\n return input"
:source-database (mt/id)
:source-tables source-tables}
diff --git a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/components/constants.ts b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/components/constants.ts
index e01451740b4..49fb86830d6 100644
--- a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/components/constants.ts
+++ b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/components/constants.ts
@@ -7,7 +7,6 @@ export const BROKEN_GROUP_TYPES: DependencyGroupType[] = [
"table",
"question",
"model",
- "transform",
];
export const BROKEN_DEPENDENTS_GROUP_TYPES: DependencyGroupType[] = [
diff --git a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/pages/DependencyDiagnosticsPage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/pages/DependencyDiagnosticsPage.unit.spec.tsx
index 1e97b3880b4..741306b6df2 100644
--- a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/pages/DependencyDiagnosticsPage.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/pages/DependencyDiagnosticsPage.unit.spec.tsx
@@ -171,7 +171,7 @@ describe("DependencyDiagnosticsPage", () => {
const { router } = setup({
mode: "broken",
nodes: CARD_NODES,
- urlParams: { groupTypes: ["table", "question", "transform"] },
+ urlParams: { groupTypes: ["table", "question"] },
});
await waitForListToLoad();
diff --git a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/routes.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/routes.unit.spec.tsx
index 5f2ae3b2558..e12861df373 100644
--- a/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/routes.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/monitor/dependency-diagnostics/routes.unit.spec.tsx
@@ -207,7 +207,7 @@ describe("dependency diagnostics section counts", () => {
]),
),
).toEqual({
- "breaking-types": ["table", "card", "transform"],
+ "breaking-types": ["table", "card"],
"breaking-card-types": ["question", "model"],
"breaking-include-personal-collections": ["true"],
"unreferenced-types": ["table", "card", "segment", "measure", "snippet"],
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-inspector/pages/TransformInspectPage/TransformInspectPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-inspector/pages/TransformInspectPage/TransformInspectPage.tsx
index fc47be84873..4ee3e380589 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-inspector/pages/TransformInspectPage/TransformInspectPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-inspector/pages/TransformInspectPage/TransformInspectPage.tsx
@@ -1,15 +1,12 @@
-import { match } from "ts-pattern";
import { t } from "ttag";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import { PageContainer } from "metabase/common/data-studio/components/PageContainer";
import { useLocation, useParams } from "metabase/router";
-import { TransformDisconnectedDatabaseBanner } from "metabase/transforms/components/TransformDisconnectedDatabaseBanner";
import { TransformHeader } from "metabase/transforms/components/TransformHeader";
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
import { useTransformWithPolling } from "metabase/transforms/hooks/use-transform-with-polling";
import { RunSection } from "metabase/transforms/pages/TransformRunPage/RunSection";
-import { isMissingSourceDatabase } from "metabase/transforms/utils";
import { Alert, Center, Icon } from "metabase/ui";
import * as Urls from "metabase/urls";
@@ -42,37 +39,25 @@ export const TransformInspectPage = () => {
return (
<PageContainer data-testid="transform-inspect-content">
<TransformHeader transform={transform} readOnly={readOnly} />
- {match({
- hasSucceeded: transform.last_run?.status === "succeeded",
- isMissingSourceDatabase: isMissingSourceDatabase(transform),
- })
- .with({ isMissingSourceDatabase: true }, () => (
- <TransformDisconnectedDatabaseBanner transform={transform} />
- ))
- .with({ hasSucceeded: false }, () => (
- <>
- <Alert
- size="compact"
- color="core-brand"
- icon={<Icon name="info" />}
- >
- {t`To inspect the transform you need to run it first.`}
- </Alert>
- <RunSection
- transform={transform}
- noTitle={true}
- readOnly={readOnly}
- permissionsReadOnly={permissionsReadOnly}
- />
- </>
- ))
- .otherwise(() => (
- <InspectorContent
+ {transform.last_run?.status !== "succeeded" ? (
+ <>
+ <Alert size="compact" color="core-brand" icon={<Icon name="info" />}>
+ {t`To inspect the transform you need to run it first.`}
+ </Alert>
+ <RunSection
transform={transform}
- params={params}
- location={location}
+ noTitle={true}
+ readOnly={readOnly}
+ permissionsReadOnly={permissionsReadOnly}
/>
- ))}
+ </>
+ ) : (
+ <InspectorContent
+ transform={transform}
+ params={params}
+ location={location}
+ />
+ )}
</PageContainer>
);
};
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 7a238147299..765c09c500c 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -43,7 +43,7 @@ export type Transform = {
can_write?: boolean;
can_execute?: boolean;
- source_database_id?: DatabaseId | null;
+ source_database_id?: DatabaseId;
// true when transform was deleted but still referenced by runs
deleted?: boolean;
diff --git a/frontend/src/metabase/admin/databases/components/ContentRemovalConfirmation/ContentRemovalConfirmation.tsx b/frontend/src/metabase/admin/databases/components/ContentRemovalConfirmation/ContentRemovalConfirmation.tsx
index b8bfd98480e..5736c5f9eab 100644
--- a/frontend/src/metabase/admin/databases/components/ContentRemovalConfirmation/ContentRemovalConfirmation.tsx
+++ b/frontend/src/metabase/admin/databases/components/ContentRemovalConfirmation/ContentRemovalConfirmation.tsx
@@ -86,8 +86,8 @@ const ContentRemovalConfirmation = ({
{transform > 0 && (
<Checkbox
label={ngettext(
- msgid`${transform} transform will stop working`,
- `${transform} transforms will stop working`,
+ msgid`Delete ${transform} transform`,
+ `Delete ${transform} transforms`,
transform,
)}
name="transform"
diff --git a/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.tsx b/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.tsx
index c91313b0ceb..7ae2157ceea 100644
--- a/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.tsx
+++ b/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.tsx
@@ -75,7 +75,7 @@ export const DatabaseDangerZoneSection = ({
<DeleteDatabaseModal
opened={isDeleteDbModalOpen}
title={t`Delete the ${database.name} database?`}
- defaultDatabaseRemovalMessage={t`This will delete every saved question, model, metric, and segment you’ve made that uses this data, and can’t be undone. Transforms that use this database won’t be deleted, but they will stop working.`}
+ defaultDatabaseRemovalMessage={t`This will delete every saved question, model, metric, segment, and transform you’ve made that uses this data, and can’t be undone!`}
database={database}
onClose={deleteDbModal.close}
onDelete={handleDeleteDatabase}
diff --git a/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.unit.spec.tsx b/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.unit.spec.tsx
index 26fe532b6a0..18d3755c1f2 100644
--- a/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.unit.spec.tsx
+++ b/frontend/src/metabase/admin/databases/components/DatabaseDangerZoneSection/DatabaseDangerZoneSection.unit.spec.tsx
@@ -176,9 +176,7 @@ describe("DatabaseDangerZoneSection", () => {
database.name,
);
await userEvent.click(
- within(modal).getByRole("button", {
- name: "Delete this DB connection",
- }),
+ within(modal).getByRole("button", { name: "Delete" }),
);
await waitFor(() => {
expect(getDiscardFieldValuesConfirmModal()).not.toBeInTheDocument();
@@ -195,7 +193,7 @@ describe("DatabaseDangerZoneSection", () => {
dataset: 2,
metric: 3,
segment: 4,
- transform: 0,
+ transform: 5,
},
});
await userEvent.click(
@@ -203,7 +201,7 @@ describe("DatabaseDangerZoneSection", () => {
);
const modal = getRemoveDatabaseConfirmModal();
const deleteButton = within(modal).getByRole("button", {
- name: "Delete this DB connection",
+ name: "Delete this content and the DB connection",
});
expect(deleteButton).toBeDisabled();
@@ -214,6 +212,7 @@ describe("DatabaseDangerZoneSection", () => {
"Delete 2 models",
"Delete 3 metrics",
"Delete 4 segments",
+ "Delete 5 transforms",
].map((label) => within(modal).findByLabelText(label)),
);
for (const checkbox of contentCheckboxes) {
@@ -223,7 +222,7 @@ describe("DatabaseDangerZoneSection", () => {
}
expect(
within(modal).getByText(
- "This will delete every saved question, model, metric, and segment you’ve made that uses this data, and can’t be undone. Transforms that use this database won’t be deleted, but they will stop working.",
+ "This will delete every saved question, model, metric, segment, and transform you’ve made that uses this data, and can’t be undone!",
),
).toBeInTheDocument();
expect(deleteButton).toBeDisabled();
diff --git a/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.tsx b/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.tsx
index e3af0f10f82..2d85eea3119 100644
--- a/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.tsx
+++ b/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.tsx
@@ -87,7 +87,9 @@ export const DeleteDatabaseModal = ({
const canConfirmName = isContentRemovalConfirmed || !hasContent;
const canDelete = canConfirmName && isDatabaseNameConfirmed;
- const deleteButtonLabel = t`Delete this DB connection`;
+ const deleteButtonLabel = hasContent
+ ? t`Delete this content and the DB connection`
+ : t`Delete`;
const errorMessage = getDeleteErrorMessage(error);
const hasMoreThanOneEntityType = usageInfo && entityTypesCount(usageInfo) > 1;
diff --git a/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.unit.spec.tsx b/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.unit.spec.tsx
index 83b9d8b620b..d65e4e9be1e 100644
--- a/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.unit.spec.tsx
+++ b/frontend/src/metabase/admin/databases/components/DeleteDatabaseModel/DeleteDatabaseModal.unit.spec.tsx
@@ -6,12 +6,12 @@ import {
screen,
waitForLoaderToBeRemoved,
} from "__support__/ui";
-import type { Database } from "metabase-types/api";
+import type { Database, DatabaseUsageInfo } from "metabase-types/api";
import type { DeleteDatabaseModalProps } from "./DeleteDatabaseModal";
import { DeleteDatabaseModal } from "./DeleteDatabaseModal";
-const getUsageInfo = (hasContent: boolean) => ({
+const getUsageInfo = (hasContent: boolean): DatabaseUsageInfo => ({
question: hasContent ? 10 : 0,
dataset: hasContent ? 20 : 0,
metric: hasContent ? 30 : 0,
@@ -22,20 +22,24 @@ const getUsageInfo = (hasContent: boolean) => ({
// Unjustified type cast. FIXME
const database = { name: "database name", id: 1 } as Database;
+type SetupOpts = {
+ onDelete?: DeleteDatabaseModalProps["onDelete"];
+ hasContent?: boolean;
+ usageInfo?: DatabaseUsageInfo;
+};
+
const setup = async ({
onDelete = jest.fn(),
hasContent = true,
-}: {
- onDelete?: DeleteDatabaseModalProps["onDelete"];
- hasContent?: boolean;
-} = {}) => {
- fetchMock.get("path:/api/database/1/usage_info", getUsageInfo(hasContent));
+ usageInfo = getUsageInfo(hasContent),
+}: SetupOpts = {}) => {
+ fetchMock.get("path:/api/database/1/usage_info", usageInfo);
renderWithProviders(
<DeleteDatabaseModal
opened
title={"Delete the destination database?"}
defaultDatabaseRemovalMessage={
- "Users routed to this database will lose access to every question, model, metric, and segment if you continue. Transforms that use this database won’t be deleted, but they will stop working."
+ "Users routed to this database will lose access to every question, model, metric, and segment if you continue."
}
onClose={jest.fn()}
onDelete={onDelete}
@@ -58,9 +62,7 @@ describe("DeleteDatabaseModal", () => {
it("should allow deleting database without content after confirming its name", async () => {
const { onDelete } = await setup({ hasContent: false });
- const deleteButton = screen.getByRole("button", {
- name: "Delete this DB connection",
- });
+ const deleteButton = screen.getByRole("button", { name: "Delete" });
expect(
screen.queryByText("Delete 10 saved questions"),
@@ -84,7 +86,7 @@ describe("DeleteDatabaseModal", () => {
const { onDelete } = await setup({ hasContent: true });
const deleteButton = screen.getByRole("button", {
- name: "Delete this DB connection",
+ name: "Delete this content and the DB connection",
});
expect(deleteButton).toBeDisabled();
@@ -93,7 +95,7 @@ describe("DeleteDatabaseModal", () => {
await userEvent.click(screen.getByText("Delete 20 models"));
await userEvent.click(screen.getByText("Delete 30 metrics"));
await userEvent.click(screen.getByText("Delete 40 segments"));
- await userEvent.click(screen.getByText("50 transforms will stop working"));
+ await userEvent.click(screen.getByText("Delete 50 transforms"));
expect(deleteButton).toBeDisabled();
@@ -109,6 +111,36 @@ describe("DeleteDatabaseModal", () => {
expect(onDelete).toHaveBeenCalled();
});
+ it("should require confirming transform removal when only transforms use the database", async () => {
+ const { onDelete } = await setup({
+ usageInfo: {
+ question: 0,
+ dataset: 0,
+ metric: 0,
+ segment: 0,
+ transform: 3,
+ },
+ });
+
+ const deleteButton = screen.getByRole("button", {
+ name: "Delete this content and the DB connection",
+ });
+ expect(
+ screen.queryByTestId("database-name-confirmation-input"),
+ ).not.toBeInTheDocument();
+
+ await userEvent.click(
+ screen.getByRole("checkbox", { name: "Delete 3 transforms" }),
+ );
+ await userEvent.type(
+ screen.getByTestId("database-name-confirmation-input"),
+ "database name",
+ );
+ await userEvent.click(deleteButton);
+
+ expect(onDelete).toHaveBeenCalledTimes(1);
+ });
+
it("shows an error if removal failed", async () => {
await setup({
hasContent: false,
@@ -117,9 +149,7 @@ describe("DeleteDatabaseModal", () => {
},
});
- const deleteButton = screen.getByRole("button", {
- name: "Delete this DB connection",
- });
+ const deleteButton = screen.getByRole("button", { name: "Delete" });
await userEvent.type(
screen.getByTestId("database-name-confirmation-input"),
"database name",
diff --git a/frontend/src/metabase/data-studio/app/pages/TransformsSectionLayout/TransformsSectionLayout.tsx b/frontend/src/metabase/data-studio/app/pages/TransformsSectionLayout/TransformsSectionLayout.tsx
index 66c5eb687b2..cc34f7fb509 100644
--- a/frontend/src/metabase/data-studio/app/pages/TransformsSectionLayout/TransformsSectionLayout.tsx
+++ b/frontend/src/metabase/data-studio/app/pages/TransformsSectionLayout/TransformsSectionLayout.tsx
@@ -4,7 +4,7 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErr
import { usePageTitle } from "metabase/hooks/use-page-title";
import { PLUGIN_TRANSFORMS } from "metabase/plugins";
import { useSelector } from "metabase/redux";
-import { Outlet, useParams } from "metabase/router";
+import { Outlet } from "metabase/router";
import { useSetting } from "metabase/settings";
import { useTransformSupportedDbs } from "metabase/transforms/hooks/use-transform-supported-dbs";
import { EnableTransformsPage } from "metabase/transforms/pages/EnableTransformsPage/EnableTransformsPage";
@@ -15,7 +15,6 @@ import { SectionLayout } from "../../components/SectionLayout";
import { NoWritableDatabasesEmptyState } from "./NoWritableDatabasesEmptyState";
export function TransformsSectionLayout() {
- const params = useParams();
usePageTitle(t`Transforms`, { titleIndex: 1 });
const shouldShowUpsell = useSelector(getShouldShowTransformsUpsell);
const isTransformsSetupComplete = useSetting("transforms-setup-complete");
@@ -29,16 +28,7 @@ export function TransformsSectionLayout() {
return <EnableTransformsPage />;
}
- // Transform-detail pages (`/data-studio/transforms/:transformId/...`) must remain reachable
- // even when no writable database exists, so the analyst can read the SQL/Python body of an
- // orphaned transform after its source DB has been deleted (GDGT-2447).
- const isTransformDetailRoute = Boolean(params.transformId);
-
- if (
- !isLoadingDatabases &&
- transformsDatabases?.length === 0 &&
- !isTransformDetailRoute
- ) {
+ if (!isLoadingDatabases && transformsDatabases?.length === 0) {
return (
<SectionLayout>
<NoWritableDatabasesEmptyState />
diff --git a/frontend/src/metabase/querying/editor/hooks/use-query-metadata/use-query-metadata.ts b/frontend/src/metabase/querying/editor/hooks/use-query-metadata/use-query-metadata.ts
index 607a4e0bdb8..1e1a89512e8 100644
--- a/frontend/src/metabase/querying/editor/hooks/use-query-metadata/use-query-metadata.ts
+++ b/frontend/src/metabase/querying/editor/hooks/use-query-metadata/use-query-metadata.ts
@@ -1,5 +1,4 @@
import { useEffect, useMemo, useRef } from "react";
-import { t } from "ttag";
import _ from "underscore";
import { useLazyGetAdhocQueryMetadataQuery } from "metabase/api";
@@ -7,8 +6,7 @@ import * as Lib from "metabase-lib";
import type Question from "metabase-lib/v1/Question";
export function useQueryMetadata(question: Question) {
- const [loadMetadata, { error, isFetching, isUninitialized }] =
- useLazyGetAdhocQueryMetadataQuery();
+ const [loadMetadata, { error }] = useLazyGetAdhocQueryMetadataQuery();
const dependenciesRef = useRef<Lib.DependentItem[]>([]);
const isSourceTableLoaded = useMemo(() => {
@@ -33,14 +31,8 @@ export function useQueryMetadata(question: Question) {
}
}, [question, loadMetadata]);
- const metadataError =
- error ??
- (!isUninitialized && !isFetching && !isSourceTableLoaded
- ? t`The source database for this query is not available`
- : undefined);
-
return {
- isLoading: !isSourceTableLoaded && (isFetching || isUninitialized),
- error: metadataError,
+ isLoading: !isSourceTableLoaded,
+ error,
};
}
diff --git a/frontend/src/metabase/transforms/components/TransformDisconnectedDatabaseBanner.tsx b/frontend/src/metabase/transforms/components/TransformDisconnectedDatabaseBanner.tsx
deleted file mode 100644
index b6cc8768bac..00000000000
--- a/frontend/src/metabase/transforms/components/TransformDisconnectedDatabaseBanner.tsx
+++ /dev/null
@@ -1,27 +0,0 @@
-import { t } from "ttag";
-
-import { Alert, Icon } from "metabase/ui";
-import type { Transform } from "metabase-types/api";
-
-import { isMissingSourceDatabase } from "../utils";
-
-export const TransformDisconnectedDatabaseBanner = ({
- transform,
-}: {
- transform: Transform;
-}) => {
- if (!isMissingSourceDatabase(transform)) {
- return null;
- }
-
- return (
- <Alert
- size="compact"
- color="error"
- icon={<Icon name="warning" />}
- data-testid="disconnected-database-banner"
- >
- {t`The database this transform depends on has been disconnected. The transform can't be edited or run.`}
- </Alert>
- );
-};
diff --git a/frontend/src/metabase/transforms/hooks/use-source-state.ts b/frontend/src/metabase/transforms/hooks/use-source-state.ts
index d304d2d5bd9..1dcfba00e6b 100644
--- a/frontend/src/metabase/transforms/hooks/use-source-state.ts
+++ b/frontend/src/metabase/transforms/hooks/use-source-state.ts
@@ -30,12 +30,6 @@ function normalizeSource(
if (source.type !== "query") {
return source;
}
- // Orphan: the source database has been deleted (e.g. a serdes-imported
- // transform whose source database is missing). The body is preserved as a
- // breadcrumb but cannot be normalized through MLv2 without a database.
- if (source.query?.database == null) {
- return source;
- }
const question = selectQuestionFromOpts(state, {
dataset_query: source.query,
diff --git a/frontend/src/metabase/transforms/pages/TransformDependenciesPage/TransformDependenciesPage.tsx b/frontend/src/metabase/transforms/pages/TransformDependenciesPage/TransformDependenciesPage.tsx
index 31e390c48b2..ad314d96d93 100644
--- a/frontend/src/metabase/transforms/pages/TransformDependenciesPage/TransformDependenciesPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformDependenciesPage/TransformDependenciesPage.tsx
@@ -7,7 +7,6 @@ import { useTransformPermissions } from "metabase/transforms/hooks/use-transform
import { Card, Center } from "metabase/ui";
import * as Urls from "metabase/urls";
-import { TransformDisconnectedDatabaseBanner } from "../../components/TransformDisconnectedDatabaseBanner";
import { TransformHeader } from "../../components/TransformHeader";
export type TransformDependenciesPageParams = {
@@ -38,7 +37,6 @@ export function TransformDependenciesPage() {
return (
<PageContainer data-testid="transforms-dependencies-content">
<TransformHeader transform={transform} readOnly={readOnly} />
- <TransformDisconnectedDatabaseBanner transform={transform} />
<PLUGIN_DEPENDENCIES.DependencyGraphPageContext.Provider
value={{
baseUrl: Urls.transformDependencies(transform.id),
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index 1997e876077..ac33bf24ba3 100644
--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -34,7 +34,6 @@ import {
buildIncrementalTarget,
getInitialValues,
} from "../../components/IncrementalTransform/form";
-import { TransformDisconnectedDatabaseBanner } from "../../components/TransformDisconnectedDatabaseBanner";
import { TransformEditor } from "../../components/TransformEditor";
import { TransformHeader } from "../../components/TransformHeader";
import { useSourceState } from "../../hooks/use-source-state";
@@ -216,7 +215,6 @@ function TransformQueryPageBody({
isEditMode={isEditMode}
readOnly={readOnly}
/>
- <TransformDisconnectedDatabaseBanner transform={transform} />
<Box
w="100%"
bg="background_page-primary"
diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx
index 1c322c92712..551d0c26d2a 100644
--- a/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/TransformRunPage.tsx
@@ -6,7 +6,6 @@ import { useTransformWithPolling } from "metabase/transforms/hooks/use-transform
import { Center } from "metabase/ui";
import * as Urls from "metabase/urls";
-import { TransformDisconnectedDatabaseBanner } from "../../components/TransformDisconnectedDatabaseBanner";
import { TransformHeader } from "../../components/TransformHeader";
import { RunSection } from "./RunSection";
@@ -39,7 +38,6 @@ export const TransformRunPage = () => {
return (
<PageContainer data-testid="transforms-run-content">
<TransformHeader transform={transform} readOnly={readOnly} />
- <TransformDisconnectedDatabaseBanner transform={transform} />
<RunSection
transform={transform}
readOnly={readOnly}
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx
index 82b48b223c8..9a70e016d2e 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsPage.tsx
@@ -5,7 +5,6 @@ import { useTransformPermissions } from "metabase/transforms/hooks/use-transform
import { Center } from "metabase/ui";
import * as Urls from "metabase/urls";
-import { TransformDisconnectedDatabaseBanner } from "../../components/TransformDisconnectedDatabaseBanner";
import { TransformHeader } from "../../components/TransformHeader";
import { useTransformWithPolling } from "../../hooks/use-transform-with-polling";
@@ -44,7 +43,6 @@ export const TransformSettingsPage = () => {
return (
<PageContainer data-testid="transforms-target-content">
<TransformHeader transform={transform} readOnly={readOnly} />
- <TransformDisconnectedDatabaseBanner transform={transform} />
<TransformSettingsSection
transform={transform}
readOnly={readOnly}
diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.ts
index 656bf0cb828..d8808e7ba95 100644
--- a/frontend/src/metabase/transforms/utils.ts
+++ b/frontend/src/metabase/transforms/utils.ts
@@ -252,10 +252,6 @@ export const isMbqlQuery = (
return !Lib.queryDisplayInfo(query).isNative;
};
-export function isMissingSourceDatabase(transform: Transform) {
- return transform.source_database_id == null;
-}
-
/**
* Returns the duration in ms of a transform run, or null when it cannot be
* measured (run still in progress, missing timestamps, unparseable values).
diff --git a/resources/migrations/2026/20261103_transform_source_database_delete_cascade.yaml b/resources/migrations/2026/20261103_transform_source_database_delete_cascade.yaml
new file mode 100644
index 00000000000..d2b037231f6
--- /dev/null
+++ b/resources/migrations/2026/20261103_transform_source_database_delete_cascade.yaml
@@ -0,0 +1,44 @@
+## Delete a transform with its source database, as a Card is deleted with its database: delete the query transforms
+## whose source database is already gone, then recreate fk_tranform_source_database_id with ON DELETE CASCADE instead
+## of SET NULL. source_database_id stays nullable, since a Python transform without source tables has no source
+## database.
+
+databaseChangeLog:
+ - changeSet:
+ id: pz7hdr
+ author: ranquild
+ comment: Delete the query transforms whose source database was deleted
+ changes:
+ - sql:
+ sql: DELETE FROM transform WHERE source_database_id IS NULL AND source_type IN ('native', 'mbql')
+ rollback:
+
+ - changeSet:
+ id: cyme8a
+ author: ranquild
+ comment: Drop fk_tranform_source_database_id (recreated next with ON DELETE CASCADE)
+ changes:
+ - dropForeignKeyConstraint:
+ baseTableName: transform
+ constraintName: fk_tranform_source_database_id
+ rollback:
+ - addForeignKeyConstraint:
+ baseTableName: transform
+ baseColumnNames: source_database_id
+ referencedTableName: metabase_database
+ referencedColumnNames: id
+ constraintName: fk_tranform_source_database_id
+ onDelete: SET NULL
+
+ - changeSet:
+ id: kq4w2m
+ author: ranquild
+ comment: Recreate fk_tranform_source_database_id with ON DELETE CASCADE
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: transform
+ baseColumnNames: source_database_id
+ referencedTableName: metabase_database
+ referencedColumnNames: id
+ constraintName: fk_tranform_source_database_id
+ onDelete: CASCADE
diff --git a/src/metabase/indexes/schema.clj b/src/metabase/indexes/schema.clj
index 88595c2c59c..2f5321d76bd 100644
--- a/src/metabase/indexes/schema.clj
+++ b/src/metabase/indexes/schema.clj
@@ -86,6 +86,11 @@
(:columns structured) (update :columns (fn [cols]
(mapv #(cond-> % (:direction %) (update :direction keyword)) cols)))))
+(def index-kinds
+ "Every `:kind` an [[::index-structured]] can have."
+ [:btree :hash :gin :gist :brin :spgist :fulltext :spatial :clustered :nonclustered :columnstore
+ :sortkey :distkey :clustering :order-by :skip-index])
+
(mr/def ::index-structured
"A single structured index definition, dispatched on `:kind`. The shape stored in
`metabase_table_indexes.structured` and handed to the driver index multimethods."
diff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.clj
index 696803f2c55..2025d442659 100644
--- a/src/metabase/permissions/models/data_permissions.clj
+++ b/src/metabase/permissions/models/data_permissions.clj
@@ -1633,17 +1633,13 @@
:full)
(defn has-db-transforms-permission?
- "Returns true if the given user has the transforms permission for the given source db.
- Superusers always pass. A nil `database-id` (an orphaned transform whose source database
- was deleted) only grants permission to superusers."
+ "Returns true if the given user has the transforms permission for the given source db."
[user-id database-id]
(and (not= database-id audit/audit-db-id)
- (or (is-superuser? user-id)
- (and (some? database-id)
- (user-has-permission-for-database? user-id
- :perms/transforms
- :yes
- database-id)))))
+ (user-has-permission-for-database? user-id
+ :perms/transforms
+ :yes
+ database-id)))
(defn has-any-transforms-permission?
"Returns true if the current user has the transforms permission for _any_ source db."
diff --git a/src/metabase/transforms/canceling.clj b/src/metabase/transforms/canceling.clj
index 356e7a8a29b..6263e2fb553 100644
--- a/src/metabase/transforms/canceling.clj
+++ b/src/metabase/transforms/canceling.clj
@@ -9,47 +9,65 @@
[metabase.run-tracking.core :as rt]
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.models.transform-run-cancelation :as wr.cancelation]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.util :as u]
[metabase.util.jvm :as u.jvm]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms])
(:import
(com.mchange.v2.resourcepool TimeoutException)
(java.time OffsetDateTime)
- (java.util.concurrent Executors ScheduledExecutorService TimeUnit)))
+ (java.util.concurrent Future Executors ScheduledExecutorService TimeUnit)
+ (org.quartz JobExecutionContext)))
(set! *warn-on-reflection* true)
(def ^:private job-key "metabase.transforms.canceling")
+(mr/def ::canceled-run
+ "A TransformRun that a cancelation request was made for, with the `:request_time` of that request."
+ [:merge
+ ::transforms.schema/transform-run
+ [:map {:closed true}
+ [:request_time ms/TemporalInstant]]])
+
+(mr/def ::outcome
+ "How a cancelation completed."
+ [:enum "success" "timeout" "error"])
+
(defonce ^:private ^ScheduledExecutorService scheduler
(Executors/newScheduledThreadPool 1))
(defonce ^:private connections (atom {}))
-(defn chan-start-run!
+(mu/defn chan-start-run! :- :nil
"Registers cancel-chan for run-id"
- [run-id cancel-chan]
+ [run-id :- ms/PositiveInt
+ cancel-chan :- ::transforms-base.schema/chan]
(swap! connections assoc run-id cancel-chan)
nil)
-(defn chan-end-run!
+(mu/defn chan-end-run! :- [:maybe ::transforms-base.schema/chan]
"Deregisters the cancel-chan for run-id and returns the channel"
- [run-id]
+ [run-id :- ms/PositiveInt]
(-> (swap-vals! connections dissoc run-id)
first ;; old value
(get run-id)))
-(defn chan-signal-cancel!
+(mu/defn chan-signal-cancel! :- [:maybe :boolean]
"Cancels the run for a given run-id"
- [run-id]
+ [run-id :- ms/PositiveInt]
(when-some [cancel-chan (chan-end-run! run-id)]
(a/put! cancel-chan :cancel!)
true))
-(defn heartbeat-and-reconcile-runs!
+(mu/defn heartbeat-and-reconcile-runs! :- :nil
"Per-node tick: heartbeat the runs this process is executing (those registered in `connections`),
then signal cancel for any that another path (timeout sweeper, reaper, force-cancel) already
terminated, so the local work stops promptly. Each node reconciles its own runs — only the
@@ -61,9 +79,10 @@
:heartbeat! transform-run/heartbeat-runs!
:on-gone chan-signal-cancel!}))
-(defn chan-start-timeout-vthread!
+(mu/defn chan-start-timeout-vthread! :- (ms/InstanceOfClass Future)
"Starts a vthread that will signal a timeout after a given number of minutes."
- [run-id timeout-minutes]
+ [run-id :- ms/PositiveInt
+ timeout-minutes :- number?]
(u.jvm/in-virtual-thread*
(try
(Thread/sleep (long (* timeout-minutes 60 1000)))
@@ -84,20 +103,25 @@
(some-> timeout-handle# future-cancel)
(chan-end-run! run-id#))))))
-(defn- request-latency-ms
+(mu/defn- request-latency-ms :- [:maybe number?]
"Milliseconds elapsed since `request-time`, or nil when `request-time` is nil.
`request-time` is the `:time` column of `transform_run_cancelation`. Metabase's app-db JDBC layer (see
`metabase.app-db.jdbc-protocols/read-column` for `Types/TIMESTAMP_WITH_TIMEZONE` and the mysql `Types/TIMESTAMP`
branch) normalizes that column to `java.time.OffsetDateTime` on every supported app DB."
- [request-time]
+ [request-time :- [:maybe ms/TemporalInstant]]
(some-> ^OffsetDateTime request-time .toInstant .toEpochMilli u/since-ms-wall-clock))
-(defn- record-completion!
+(mu/defn- record-completion! :- [:maybe [:map {:closed true}
+ [:object [:or ::transforms.schema/transform-run ::canceled-run]]
+ [:details [:map {:closed true} [:outcome ::outcome]]]]]
"Emit metrics + audit event for a cancelation completion. `outcome` ∈ #{\"success\" \"timeout\" \"error\"}.
`run` is the pre-fetched `:model/TransformRun` row (or nil); when nil the audit event is skipped, metrics still
emit. The caller owns the fetch so we don't redundantly refetch a row it already has."
- [run-id run request-time outcome]
+ [run-id :- ms/PositiveInt
+ run :- [:maybe [:or ::transforms.schema/transform-run ::canceled-run]]
+ request-time :- [:maybe ms/TemporalInstant]
+ outcome :- ::outcome]
(analytics/inc! :metabase-transforms/cancelation-completed {:outcome outcome})
(when-let [latency (request-latency-ms request-time)]
(analytics/observe! :metabase-transforms/cancelation-latency-ms
@@ -111,14 +135,15 @@
(log/warnf "Failed to publish transform-run-canceled event for run %s (outcome=%s): %s"
run-id outcome (ex-message t)))))
-(defn cancel-run!
+(mu/defn cancel-run! :- [:maybe :int]
"Cancel a `run` (a `:model/TransformRun` row). `request-time` is the `java.time.OffsetDateTime` of the cancelation
request — the background loop passes the `:time` from the cancelation row; the API path passes
`(OffsetDateTime/now)` since it just inserted the row microseconds earlier.
Callers own the fetching of `run` since the full row is needed for instrumentation & audit logging of this
operation which changes state."
- [{run-id :id :as run} request-time]
+ [{run-id :id :as run} :- ::transforms.schema/transform-run
+ request-time :- ms/TemporalInstant]
(tracing/with-span :tasks "task.transform.cancel" {:run/id run-id
:transform/id (:transform_id run)}
(try
@@ -131,7 +156,8 @@
(record-completion! run-id run request-time "error")
(throw t)))))
-(defn- cancel-old-transform-runs! [_ctx]
+(mu/defn- cancel-old-transform-runs! :- :nil
+ [_ctx :- (ms/InstanceOfClass JobExecutionContext)]
;; `cancel-old-canceling-runs!` does a transactional select-then-update-by-id under row locks, so it returns
;; *only* the runs this sweep actually transitioned — `outcome=timeout` cannot overcount rows another path
;; (cancel-run!, timeout-run!) had already finished. "timeout" here = the *cancelation attempt* exceeded 2 min;
@@ -149,7 +175,8 @@
(log/errorf "Error force-canceling stale transform runs: %s" (ex-message t))
;; Transaction rolled back — the row count is unrecoverable, so emit one aggregate error so the failure
;; is at least visible in metrics. Magnitude can be inferred from log volume if needed.
- (analytics/inc! :metabase-transforms/cancelation-completed {:outcome "error"}))))
+ (analytics/inc! :metabase-transforms/cancelation-completed {:outcome "error"})
+ nil)))
(task/defjob CancelOldTransformRuns
"Cancel items that haven't been canceled in two minutes"
@@ -158,7 +185,8 @@
[ctx]
(cancel-old-transform-runs! ctx))
-(defn- start-job! []
+(mu/defn- start-job! :- [:maybe inst?]
+ []
(when (not (task/job-exists? job-key))
(let [job (jobs/build
(jobs/of-type CancelOldTransformRuns)
@@ -186,10 +214,10 @@
:label "transform run"
:reap-fn #(transform-run/reap-orphaned-runs! transform-heartbeat-stale-minutes)}))
-(defn- app-db-checkout-timeout?
+(mu/defn- app-db-checkout-timeout? :- :boolean
"Is `e`, or anything in its cause chain, a c3p0 pool-checkout timeout? Then the app DB pool is exhausted, and this
poller is only the messenger."
- [^Throwable e]
+ [^Throwable e :- (ms/InstanceOfClass Throwable)]
(boolean (some #(instance? TimeoutException %) (u/full-exception-chain e))))
(defmethod task/init! ::CancelRuns [_]
diff --git a/src/metabase/transforms/coordinated_run.clj b/src/metabase/transforms/coordinated_run.clj
index d5ffb2038df..7192767523f 100644
--- a/src/metabase/transforms/coordinated_run.clj
+++ b/src/metabase/transforms/coordinated_run.clj
@@ -5,65 +5,88 @@
`model`."
(:require
[metabase.run-tracking.core :as rt]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms.canceling :as canceling]
[metabase.transforms.db :as transforms.db]
- [metabase.transforms.models.transform-run-cancelation :as transform-run-cancelation]))
+ [metabase.transforms.models.transform-run-cancelation :as transform-run-cancelation]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
-(defn- touch-active-run!
- [model run-id]
+(mr/def ::run-changes
+ "Changes to a row of a coordinated-run model."
+ [:or
+ ::transforms.schema/transform-job-run.update
+ ::transforms.schema/transform-dag-run.update])
+
+(mu/defn- touch-active-run! :- :int
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt]
(case model
:model/TransformJobRun (transforms.db/touch-active-job-run! run-id)
:model/TransformDagRun (transforms.db/touch-active-dag-run! run-id)))
-(defn- finish-active-run!
- [model run-id changes]
+(mu/defn- finish-active-run! :- :int
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt
+ changes :- ::run-changes]
(case model
:model/TransformJobRun (transforms.db/finish-active-job-run! run-id changes)
:model/TransformDagRun (transforms.db/finish-active-dag-run! run-id changes)))
-(defn add-run-activity!
+(mu/defn add-run-activity! :- :int
"Note that a run has had activity (touches `updated_at`)."
- [model run-id]
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt]
(touch-active-run! model run-id))
-(defn succeed-started-run!
+(mu/defn succeed-started-run! :- :int
"Mark a started run as successfully completed."
- ([model run-id]
+ ([model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt]
(succeed-started-run! model run-id {}))
- ([model run-id properties]
+ ([model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt
+ properties :- ::run-changes]
(finish-active-run! model
run-id
(merge properties
{:status :succeeded
:is_active nil}))))
-(defn fail-started-run!
+(mu/defn fail-started-run! :- :int
"Mark the started active run as failed and inactive."
- [model run-id properties]
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt
+ properties :- ::run-changes]
(finish-active-run! model
run-id
(merge properties
{:status :failed
:is_active nil})))
-(defn cancel-started-run!
+(mu/defn cancel-started-run! :- :int
"Mark an active run as canceled; a finished run is never resurrected into a canceled state.
Returns the number of rows updated — 0 if the run had already finished."
- [model run-id]
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt]
(finish-active-run! model
run-id
{:status :canceled
:is_active nil
:message "Canceled"}))
-(defn cancel!
+(mu/defn cancel! :- :boolean
"Cancel an in-progress coordinated run: mark it canceled and request cancellation of its
still-active member transform runs — via the in-process cancel channel, plus a cancelation row so
the cancel-runs task reaches members running on other nodes. Returns true if the run was active
and is now canceled, false if it had already finished."
- [model member-fk run-id]
+ [model :- ::transforms.schema/coordinated-run-model
+ member-fk :- [:enum :job_run_id :dag_run_id]
+ run-id :- ms/PositiveInt]
(boolean
(when (pos? (cancel-started-run! model run-id))
(doseq [member-run-id (transforms.db/active-run-ids-of-parent member-fk run-id)]
@@ -71,16 +94,19 @@
(canceling/chan-signal-cancel! member-run-id))
true)))
-(defn heartbeat-runs!
+(mu/defn heartbeat-runs! :- [:maybe [:sequential :int]]
"Stamp `last_heartbeat = now` on the given still-active run-ids."
- [model run-ids]
+ [model :- ::transforms.schema/coordinated-run-model
+ run-ids :- [:sequential ms/PositiveInt]]
(rt/heartbeat-ids! model [:is_active true] :last_heartbeat run-ids))
-(defn reap-orphaned-runs!
+(mu/defn reap-orphaned-runs! :- [:maybe [:sequential [:or ::transforms.schema/transform-job-run ::transforms.schema/transform-dag-run]]]
"Time out active runs whose `last_heartbeat` is older than `stale-minutes` (their coordinator
process is presumed dead). `type-tag` (\"job\"/\"dag\") tags the emitted metrics. Returns the rows
that were timed out so callers can notify."
- [model type-tag stale-minutes]
+ [model :- ::transforms.schema/coordinated-run-model
+ type-tag :- [:enum "job" "dag"]
+ stale-minutes :- ms/PositiveInt]
(rt/reap-orphaned!
{:model model
:active [:is_active true]
@@ -92,11 +118,12 @@
:latency-column :last_heartbeat
:timeout-ms (rt/unit->ms stale-minutes :minute)}}))
-(defn heartbeat-and-reconcile!
+(mu/defn heartbeat-and-reconcile! :- :nil
"Stamp a heartbeat on every active run whose id is a key of `active-runs-atom` (the runs this
process coordinates), then deliver the `gone` promise of any that were terminated externally so
their coordinator aborts."
- [model active-runs-atom]
+ [model :- ::transforms.schema/coordinated-run-model
+ active-runs-atom :- ::transforms-base.schema/atom]
(rt/heartbeat-and-reconcile! {:model model
:active [:is_active true]
:ids (keys @active-runs-atom)
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index dd301e4b20e..cf135f9f4c3 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -41,6 +41,7 @@
is-temp-transform-table?]
[metabase.transforms.crud
check-database-feature
+ requestable-indexes
check-feature-enabled!
validate-incremental-column-type!
validate-target-schema!
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index b0602578aab..8007bf793b9 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -9,25 +9,27 @@
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.types.isa :as lib.types.isa]
- [metabase.models.interface :as mi]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as transforms-base.ordering]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.feature-gating :as transforms.gating]
[metabase.transforms.models.transform :as transform.model]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
- [metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn check-database-feature
+(mu/defn check-database-feature :- :nil
"Check that the target database supports the required features for this transform."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(let [database (api/check-400 (transforms.db/database (transforms-base.i/target-db-id transform))
(deferred-tru "The target database cannot be found."))
features (transforms-base.u/required-database-features transform)]
@@ -38,45 +40,48 @@
(api/check-400 (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(deferred-tru "The database does not support the requested transform features."))
(api/check-400 (not (database-routing/db-routing-enabled? (driver.u/lib-database (:id database))))
- (deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
+ (deferred-tru "Transforms are not supported on databases with DB routing enabled."))
+ nil))
-(defn check-feature-enabled!
+(mu/defn check-feature-enabled! :- :nil
"Check that the premium features required for this transform type are enabled."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(api/check (transforms.u/check-feature-enabled transform)
- [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
+ [402 (deferred-tru "Premium features required for this transform type are not enabled.")])
+ nil)
-(defn validate-transform-query!
+(mu/defn validate-transform-query! :- :nil
"Validate that the transform query is valid. Throws a 400 error if validation fails."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(when-let [error (transforms-base.u/validate-transform-query transform)]
(throw (ex-info (:error error)
(assoc error
:status-code 400)))))
-(defn validate-target-schema!
+(mu/defn validate-target-schema! :- :nil
"Require a non-blank `:target.schema` when the target database supports schemas.
On schemas-supporting drivers a nil schema makes post-run sync miss the physical table
and leaves the Metabase table with zero fields."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(let [db-id (transforms-base.i/target-db-id transform)
db (transforms.db/database db-id)]
(when (and db (driver.u/supports? (:engine db) :schemas db))
(api/check-400 (not (str/blank? (get-in transform [:target :schema])))
- (deferred-tru "A target schema is required for this database.")))))
+ (deferred-tru "A target schema is required for this database.")))
+ nil))
-(defn- field->column
+(mu/defn- field->column :- :metabase.transforms-base.schema/column-types
"Adapt a t2 Field row to the kebab-case column shape `lib.types.isa` predicates expect."
- [field]
+ [field :- :metabase.warehouse-schema.schema/field]
{:base-type (:base_type field), :effective-type (:effective_type field)})
-(defn validate-incremental-column-type!
+(mu/defn validate-incremental-column-type! :- :nil
"Validates that the checkpoint column for an incremental transform has a supported type.
Resolves the field by ID and checks that its base-type is numeric or temporal.
Throws a 400 error if the field cannot be found or its type is not supported."
- [{:keys [source] :as transform}]
+ [{:keys [source] :as transform} :- ::transforms.schema/transform.definition]
(when (transforms-base.u/checkpoint-source? transform)
(let [checkpoint-filter-field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
field (transforms.db/field checkpoint-filter-field-id)]
@@ -85,12 +90,13 @@
(api/check-400 (transforms-base.u/supported-checkpoint-column? column)
(deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
(:name field)
- (pr-str (lib.types.isa/column-type column))))))))
+ (pr-str (lib.types.isa/column-type column)))))))
+ nil)
-(defn validate-lookback!
+(mu/defn validate-lookback! :- :nil
"Validates a configured lookback window: only date and datetime checkpoint columns support one
(time-only columns wrap at midnight), and date-only columns need a day-or-coarser unit."
- [{:keys [source] :as transform}]
+ [{:keys [source] :as transform} :- ::transforms.schema/transform.definition]
(let [{:keys [lookback checkpoint-filter-field-id]} (:source-incremental-strategy source)]
(when-let [{:keys [unit]} (and (transforms-base.u/checkpoint-source? transform) lookback)]
(when-let [field (transforms.db/field checkpoint-filter-field-id)]
@@ -99,24 +105,31 @@
(deferred-tru "A lookback window is only supported for date or datetime checkpoint columns."))
(when (lib.types.isa/date-without-time? column)
(api/check-400 (contains? #{"day" "week" "month" "quarter" "year"} unit)
- (deferred-tru "A lookback window on a date checkpoint column must use days or a coarser unit."))))))))
+ (deferred-tru "A lookback window on a date checkpoint column must use days or a coarser unit.")))))))
+ nil)
-(defn validate-incremental-table-tag!
+(mu/defn validate-incremental-table-tag! :- :nil
"Reject a table-incremental native-query transform whose source query has no table template tag for
the incremental range filter to target.
Without that table variable the incremental filter has nowhere to be injected, so the transform is
in a broken state. This surfaces the error eagerly at create/update time instead of only when the
transform is run."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(when (and (transforms-base.u/incremental-target? transform)
(transforms-base.u/native-query-transform? transform))
(api/check-400 (transforms-base.u/incremental-table-tag-name transform)
- (deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))))
+ (deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))
+ nil))
-(defn get-transforms
+(mu/defn get-transforms :- [:sequential ::transforms.schema/transform]
"Get a list of transforms."
- [& {:keys [last-run-start-time last-run-statuses tag-ids database-id]}]
+ [& {:keys [last-run-start-time last-run-statuses tag-ids database-id]}
+ :- [:maybe [:map {:closed true}
+ [:last-run-start-time {:optional true} [:maybe ms/NonBlankString]]
+ [:last-run-statuses {:optional true} [:maybe [:sequential [:enum "started" "succeeded" "failed" "timeout"]]]]
+ [:tag-ids {:optional true} [:maybe [:sequential ms/IntGreaterThanOrEqualToZero]]]
+ [:database-id {:optional true} [:maybe ms/PositiveInt]]]]]
(let [enabled-types (transforms.gating/enabled-source-types-for-user)]
(api/check-403 (seq enabled-types))
(let [transforms (transforms.db/transforms-of-source-types enabled-types database-id)]
@@ -125,12 +138,11 @@
(comp (transforms-base.u/->date-field-filter-xf [:last_run :start_time] last-run-start-time)
(transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)
(transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)
- (map #(update % :last_run transforms-base.u/present-run))
(map transforms.u/add-source-readable)))))))
-(defn- requestable-indexes
+(mu/defn requestable-indexes :- [:maybe ::transforms.schema/requestable-indexes]
"The index methods the target db's driver can create on `transform`'s target table, or nil when none are available."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(when-let [db-id (transforms-base.i/target-db-id transform)]
(when-let [database (when (transforms.db/database-exists? db-id)
(driver.u/lib-database db-id))]
@@ -141,29 +153,29 @@
nil))]
(not-empty methods)))))
-(defn get-transform
+(mu/defn get-transform :- ::transforms.schema/transform
"Get a specific transform."
- [id]
+ [id :- ::lib.schema.id/transform]
(let [{:keys [target] :as transform} (api/read-check :model/Transform id)
target-table (transforms-base.u/target-table (transforms-base.i/target-db-id transform) target :active true)]
(-> transform
(t2/hydrate :last_run :transform_tag_ids :creator :owner :can_read :can_write :can_execute)
- (u/update-some :last_run transforms-base.u/present-run)
(assoc :table target-table)
- (assoc :requestable_indexes (requestable-indexes transform))
transforms.u/add-source-readable)))
-(defn create-transform!
+(mu/defn create-transform! :- ::transforms.schema/transform
"Create new transform in the appdb.
Optionally accepts a creator-id to use instead of the current user."
- ([body]
+ ([body :- ::transforms.schema/transform.create]
(create-transform! body nil))
- ([body creator-id]
- (when (transforms-base.u/query-transform? body)
- (validate-transform-query! body))
- (validate-target-schema! body)
- (validate-incremental-table-tag! body)
- (validate-lookback! body)
+ ([body :- ::transforms.schema/transform.create
+ creator-id :- [:maybe ::lib.schema.id/user]]
+ (let [definition (transforms.u/->definition body)]
+ (when (transforms-base.u/query-transform? definition)
+ (validate-transform-query! definition))
+ (validate-target-schema! definition)
+ (validate-incremental-table-tag! definition)
+ (validate-lookback! definition))
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
(let [tag-ids (:tag_ids body)
@@ -183,28 +195,30 @@
(events/publish-event! :event/transform-create {:object transform :user-id creator-id})
transform)))
-(defn update-transform!
+(mu/defn update-transform! :- ::transforms.schema/transform
"Update a transform. Validates features, database support, cycles, and target conflicts.
Returns the updated transform with hydrated associations."
- [id body]
+ [id :- ::lib.schema.id/transform
+ body :- ::transforms.schema/transform.create]
(let [transform (t2/with-transaction [_]
;; Cycle detection should occur within the transaction to avoid race
- (let [old (transforms.db/transform id)
- new (merge old body)
+ (let [old (api/read-check :model/Transform id)
+ new (merge old body)
+ definition (transforms.u/->definition new)
target-fields #(-> % :target (select-keys [:schema :name]))]
- (api/check-403 (and (mi/can-write? old) (mi/can-write? new)))
+ (api/update-check old body)
;; we must validate on a full transform object
- (check-feature-enabled! new)
- (check-database-feature new)
+ (check-feature-enabled! definition)
+ (check-database-feature definition)
(when (contains? body :target)
- (validate-target-schema! new))
+ (validate-target-schema! definition))
(when (contains? body :source)
- (validate-incremental-column-type! new))
+ (validate-incremental-column-type! definition))
(when (or (contains? body :source) (contains? body :target))
- (validate-incremental-table-tag! new)
- (validate-lookback! new))
+ (validate-incremental-table-tag! definition)
+ (validate-lookback! definition))
(when (transforms-base.u/query-transform? old)
- (validate-transform-query! new)
+ (validate-transform-query! definition)
(when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]
(throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
{:status-code 400}))))
@@ -221,9 +235,9 @@
(-> transform
transforms.u/add-source-readable)))
-(defn delete-transform!
+(mu/defn delete-transform! :- :nil
"Delete a transform and publish the delete event."
- [transform]
+ [transform :- ::transforms.schema/transform]
(transforms.db/delete-transform! (:id transform))
(events/publish-event! :event/transform-delete
{:object transform
diff --git a/src/metabase/transforms/dag.clj b/src/metabase/transforms/dag.clj
index 91792c603da..68bb43165d0 100644
--- a/src/metabase/transforms/dag.clj
+++ b/src/metabase/transforms/dag.clj
@@ -6,6 +6,7 @@
transform runs via `transform_run.dag_run_id`."
(:require
[clojure.set :as set]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.run-tracking.core :as rt]
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
@@ -14,8 +15,12 @@
[metabase.transforms.db :as transforms.db]
[metabase.transforms.jobs :as jobs]
[metabase.transforms.models.dag-run :as dag-run]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
@@ -23,6 +28,14 @@
"A DAG run whose coordinator hasn't heartbeat in this many minutes is presumed dead and reaped."
5)
+(mr/def ::direction
+ "Which transforms a DAG reprocess includes relative to its seed."
+ [:enum :upstream :downstream])
+
+(mr/def ::graph
+ "A graph of transform IDs to the IDs of their neighbors."
+ [:map-of ::lib.schema.id/transform [:set ::lib.schema.id/transform]])
+
(defonce ^:private dag-active-runs
;; dag-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped) so
;; its coordinator aborts. Separate from the job-run atom in [[metabase.transforms.jobs]] because
@@ -31,17 +44,18 @@
;;; ------------------------------------------- Planning -------------------------------------------
-(defn- dependents-graph
+(mu/defn- dependents-graph :- ::graph
"Reverse a forward dependency map (id -> #{deps it reads}) into a dependents map
(id -> #{transforms that read it})."
- [forward-deps]
+ [forward-deps :- ::graph]
(reduce-kv (fn [acc id deps]
(reduce #(update %1 %2 (fnil conj #{}) id) acc deps))
{} forward-deps))
-(defn- reachable
+(mu/defn- reachable :- [:set ::lib.schema.id/transform]
"Set of nodes reachable from `start` in `graph` (id -> #{neighbor-ids}), including `start`."
- [graph start]
+ [graph :- ::graph
+ start :- ::lib.schema.id/transform]
(loop [queue [start] seen #{}]
(if-let [node (first queue)]
(if (seen node)
@@ -49,18 +63,21 @@
(recur (into (rest queue) (get graph node)) (conj seen node)))
seen)))
-(defn- lean-transforms
+(mu/defn- lean-transforms :- [:sequential ::transforms.schema/transform]
"Transforms with only the columns the ordering walk needs — avoids loading every transform's full
row just to compute the dependency graph (full rows are fetched only for the resulting closure)."
[]
(transforms.db/transform-dependency-rows))
-(defn- full-transforms [ids]
+(mu/defn- full-transforms :- [:sequential ::transforms.schema/transform]
+ [ids :- [:set ::lib.schema.id/transform]]
(if (seq ids)
(transforms.db/transforms ids)
[]))
-(defn- dag-run-plan
+(mu/defn- dag-run-plan :- [:map {:closed true}
+ [:transform-ids [:set ::lib.schema.id/transform]]
+ [:plan ::transforms.schema/transform-plan]]
"Compute both the set of transform-ids a DAG reprocess should run and the execution plan to run
them. Returns `{:transform-ids #{...} :plan {:order [...] :deps {...}}}`.
@@ -74,7 +91,8 @@
the dependency edges that stay within the downstream set. Dropping the external edges
preserves ordering (the seed still runs before anything that depends on it) while
leaving those inputs untouched."
- [seed-id direction]
+ [seed-id :- ::lib.schema.id/transform
+ direction :- ::direction]
(let [lean (lean-transforms)]
(case direction
:upstream
@@ -95,25 +113,29 @@
{:transform-ids downstream-ids
:plan (jobs/dependencies->plan deps (full-transforms downstream-ids))}))))
-(defn dag-run-transforms
+(mu/defn dag-run-transforms :- [:sequential ::transforms.schema/transform]
"Return the transforms a DAG reprocess from `transform-id` would run, in planned execution order.
`direction` is as in [[run-dag!]]."
- [transform-id direction]
+ [transform-id :- ::lib.schema.id/transform
+ direction :- ::direction]
(-> (dag-run-plan transform-id direction) :plan :order vec))
;;; ------------------------------------------- Execution -------------------------------------------
-(defn- start-dag-run-unless-racing!
+(mu/defn- start-dag-run-unless-racing! :- [:maybe ::transforms.schema/transform-dag-run]
"Insert the DAG run row, or return nil if a concurrent trigger for the same seed transform won
the `(source_transform_id, is_active)` uniqueness race."
- [transform-id direction user-id transform-count]
+ [transform-id :- ::lib.schema.id/transform
+ direction :- ::direction
+ user-id :- [:maybe ::lib.schema.id/user]
+ transform-count :- :int]
(try
(dag-run/start-dag-run! transform-id direction user-id transform-count)
(catch Exception e
(when-not (transforms.u/duplicate-key-violation? e)
(throw e)))))
-(defn run-dag!
+(mu/defn run-dag! :- [:maybe ms/PositiveInt]
"Run all transforms in the dependency DAG rooted at `transform-id`, tracking progress in a
`transform_dag_run` row. Blocks until the run finishes; runs are always manual.
@@ -124,7 +146,11 @@
`:start-promise`, if provided, is delivered `[:started dag-run-id]` once the run row is created,
`nil` if nothing was run (already running / empty closure), or a Throwable on a pre-start failure.
Returns the dag-run-id, or nil if nothing was executed."
- [transform-id {:keys [direction user-id start-promise]}]
+ [transform-id :- ::lib.schema.id/transform
+ {:keys [direction user-id start-promise]} :- [:map {:closed true}
+ [:direction ::direction]
+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:start-promise {:optional true} [:maybe ::transforms.schema/promise]]]]
(try
(if (dag-run/running-run-for-source-transform-id transform-id)
(do (log/info "Not executing DAG run for transform" (pr-str transform-id) "because one is already running")
@@ -156,21 +182,21 @@
(some-> start-promise (deliver t))
(throw t))))
-(defn cancel-dag-run!
+(mu/defn cancel-dag-run! :- :boolean
"Cancel an in-progress DAG run and request cancellation of its still-running member transform runs.
Returns true if the run was active and is now canceled, false if it had already finished."
- [run-id]
+ [run-id :- ms/PositiveInt]
(coordinated-run/cancel! :model/TransformDagRun :dag_run_id run-id))
;;; ------------------------------------------- Heartbeat / reaping -------------------------------------------
-(defn- heartbeat-and-reconcile-runs!
+(mu/defn- heartbeat-and-reconcile-runs! :- :nil
"Stamp a heartbeat on every active DAG run this process is coordinating, then deliver the `gone`
promise of any that were terminated externally so their coordinator aborts."
[]
(coordinated-run/heartbeat-and-reconcile! :model/TransformDagRun dag-active-runs))
-(defn- reap-orphaned-runs!
+(mu/defn- reap-orphaned-runs! :- [:maybe [:sequential ::transforms.schema/transform-dag-run]]
"Reap DAG runs whose coordinator process died (stale heartbeat)."
[]
(coordinated-run/reap-orphaned-runs! :model/TransformDagRun "dag" dag-heartbeat-stale-minutes))
diff --git a/src/metabase/transforms/db.clj b/src/metabase/transforms/db.clj
index d3b34631435..ab288e39553 100644
--- a/src/metabase/transforms/db.clj
+++ b/src/metabase/transforms/db.clj
@@ -3,6 +3,8 @@
additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods,
and transactions."
(:require
+ [malli.core :as mc]
+ [malli.transform :as mtx]
[malli.util :as mut]
[medley.core :as m]
[metabase.app-db.core :as mdb]
@@ -10,6 +12,7 @@
[metabase.transforms.schema :as transforms.schema]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -311,8 +314,12 @@
(defn- label-case
"A Honey SQL `:case` expression translating each value of `test-column` per `labels` (a map of raw value to
display label), falling back to `fallback` (`test-column` itself, by default) when it matches none of them."
- ([test-column labels] (label-case test-column labels test-column))
- ([test-column labels fallback]
+ ([test-column
+ labels]
+ (label-case test-column labels test-column))
+ ([test-column
+ labels
+ fallback]
(-> [:case]
(into (mapcat (fn [[value label]] [[:= test-column value] label])) labels)
(conj :else fallback))))
@@ -336,7 +343,11 @@
(defn- paged-runs-order-by
"Builds a `:order-by` clause for the paged run listing, translating display values for sortable columns per
`status-labels`/`run-method-labels`/`tag-name-labels` (maps of raw value to display label)."
- [sort-column sort-direction status-labels run-method-labels tag-name-labels]
+ [sort-column
+ sort-direction
+ status-labels
+ run-method-labels
+ tag-name-labels]
(let [sort-column (or (keyword sort-column) :start-time)
sort-direction (or (keyword sort-direction) :desc)
sort-direction-with-nulls (case sort-direction
@@ -361,7 +372,7 @@
[:end_time sort-direction-with-nulls]])
[:transform_run.id sort-direction])))
-(def ^:private RunFilters
+(mr/def ::run-filters
[:map {:closed true}
[:started-at-start [:maybe ms/TemporalInstant]]
[:started-at-end [:maybe ms/TemporalInstant]]
@@ -377,7 +388,7 @@
"Up to `limit` (offset by `offset`) TransformRuns matching `filters` (see [[paged-runs-where]] for the supported
keys), sorted by `sort-column`/`sort-direction` (translating `status`, `run-method`, and `transform-tags` sort
columns per `status-labels`/`run-method-labels`/`tag-name-labels`)."
- [filters :- RunFilters
+ [filters :- ::run-filters
sort-column :- [:maybe [:or :keyword :string]]
sort-direction :- [:maybe [:or :keyword :string]]
status-labels :- [:map-of :string :string]
@@ -398,7 +409,7 @@
(mu/defn paged-run-count
"The number of TransformRuns matching `filters` (see [[paged-runs-where]] for the supported keys)."
- [filters :- RunFilters]
+ [filters :- ::run-filters]
(t2/count :model/TransformRun (m/assoc-some {} :where (paged-runs-where filters))))
(mu/defn latest-runs-reducible
@@ -561,7 +572,11 @@
(t2/update! :model/TransformDagRun :id run-id :is_active true changes))
(defn- job-run-where
- [job-id status run-method started-at-start started-at-end]
+ [job-id
+ status
+ run-method
+ started-at-start
+ started-at-end]
(let [conditions (cond-> []
job-id (conj [:= :job_id job-id])
status (conj [:= :status status])
@@ -573,7 +588,8 @@
(into [:and] conditions))))
(defn- job-run-order-by
- [sort-column sort-direction]
+ [sort-column
+ sort-direction]
(let [sort-direction (or (keyword sort-direction) :desc)
sort-direction-with-nulls (case sort-direction
:asc :asc-nulls-last
@@ -673,7 +689,8 @@
;; literals rather than plain strings: a string compiles to a `?` parameter, and H2 cannot work out the type of
;; a parameter in the select list of a derived table (`Unknown data type: "RUN_TYPE"`).
-(defn- job-run-subquery [transform-ids]
+(defn- job-run-subquery
+ [transform-ids]
^:allow-subquery
{:select [[(h2x/literal "job") :run_type]
:id
@@ -694,7 +711,8 @@
[:in :member.transform_id transform-ids]]}]
true)})
-(defn- dag-run-subquery [transform-ids]
+(defn- dag-run-subquery
+ [transform-ids]
^:allow-subquery
{:select [[(h2x/literal "dag") :run_type]
:id
@@ -714,7 +732,8 @@
[:in :member.transform_id transform-ids]]}]
true)})
-(defn- transform-run-subquery [transform-ids]
+(defn- transform-run-subquery
+ [transform-ids]
^:allow-subquery
{:select [[(h2x/literal "transform") :run_type]
:id
@@ -735,7 +754,8 @@
(defn- union-subquery
"The UNION ALL of the branches selected by `types` (a subset of `#{:job :dag :transform}`), each optionally
narrowed to runs touching one of `transform-ids`."
- [types transform-ids]
+ [types
+ transform-ids]
(let [types (set (or (seq types) #{:job :dag :transform}))]
^:allow-subquery
{:union-all (cond-> []
@@ -744,7 +764,12 @@
(:transform types) (conj (transform-run-subquery transform-ids)))}))
(defn- root-run-summaries-where
- [statuses run-methods started-at-start started-at-end ended-at-start ended-at-end]
+ [statuses
+ run-methods
+ started-at-start
+ started-at-end
+ ended-at-start
+ ended-at-end]
(let [where (into [:and] (remove nil?)
[(when (seq statuses) [:in :status (set statuses)])
;; started ⇒ still active, as in the per-table run listings
@@ -761,7 +786,8 @@
anything else — including nil — falls back to ordering by start_time then end_time) in `sort-direction`
(`:asc`/`:desc`, defaulting to `:desc`), with in-progress rows (null `end_time`) ordered as the most recent: last
ascending, first descending."
- [sort-column sort-direction]
+ [sort-column
+ sort-direction]
(let [sort-direction (or (keyword sort-direction) :desc)
sort-direction-with-nulls (case sort-direction
:asc :asc-nulls-last
@@ -772,12 +798,12 @@
[[:start_time sort-direction]
[:end_time sort-direction-with-nulls]])))
-(mu/defn root-run-summaries-page
+(mu/defn root-run-summaries-page :- [:sequential ::transforms.schema/run-summary]
"Up to `limit` (offset by `offset`) root-run summary rows -- see [[metabase.transforms.run-listing]] -- of `types`
(a subset of `#{:job :dag :transform}`, or all three when empty), optionally narrowed to `statuses`,
`run-methods`, started in [`started-at-start`, `started-at-end`), ended in [`ended-at-start`, `ended-at-end`),
and/or touching one of `transform-ids`, sorted by `sort-column`/`sort-direction`."
- [types :- [:maybe [:sequential :keyword]]
+ [types :- [:maybe [:sequential [:enum :job :dag :transform]]]
statuses :- [:maybe [:sequential :string]]
run-methods :- [:maybe [:sequential :string]]
started-at-start :- [:maybe ms/TemporalInstant]
@@ -785,18 +811,20 @@
ended-at-start :- [:maybe ms/TemporalInstant]
ended-at-end :- [:maybe ms/TemporalInstant]
transform-ids :- [:maybe [:sequential ::lib.schema.id/transform]]
- sort-column :- [:maybe [:or :keyword :string]]
- sort-direction :- [:maybe [:or :keyword :string]]
+ sort-column :- [:maybe [:enum :start_time :end_time]]
+ sort-direction :- [:maybe [:enum :asc :desc]]
limit :- ms/PositiveInt
offset :- ms/IntGreaterThanOrEqualToZero]
(let [where (root-run-summaries-where statuses run-methods started-at-start started-at-end ended-at-start
ended-at-end)
base (cond-> {:from [[(union-subquery types transform-ids) :runs]]}
where (assoc :where where))]
- (t2/query (merge base {:select [:*]
- :order-by (root-run-order-by sort-column sort-direction)
- :limit limit
- :offset offset}))))
+ (mapv (mr/cached ::run-summary-decoder ::transforms.schema/run-summary
+ (fn [] (mc/decoder ::transforms.schema/run-summary (mtx/json-transformer))))
+ (t2/query (merge base {:select [:*]
+ :order-by (root-run-order-by sort-column sort-direction)
+ :limit limit
+ :offset offset})))))
(mu/defn root-run-summaries-count
"The number of root-run summary rows matching the same filters as [[root-run-summaries-page]]."
diff --git a/src/metabase/transforms/execute.clj b/src/metabase/transforms/execute.clj
index e8175837019..0c9f7ea237c 100644
--- a/src/metabase/transforms/execute.clj
+++ b/src/metabase/transforms/execute.clj
@@ -1,35 +1,39 @@
(ns metabase.transforms.execute
(:require
[metabase.indexes.models.table-index :as table-index]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.interface :as transforms.i]
;; load the `transforms.i/execute!` `:query` method
[metabase.transforms.query-impl]
- [metabase.util.log :as log]))
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
-(defn- resolve-target
+(mu/defn- resolve-target :- ::transforms-base.schema/transform
"Hydrates the declared indexes and their request ids onto the target so the base reads them off
`(:indexes target)` and `(:index-request-ids target)` at every table-creation seam, and stashes
`:full-incremental-run?` so that (DB-backed) decision is made once and stays stable for the whole run."
- [transform]
+ [transform :- ::transforms.schema/transform]
(let [requests (table-index/select-applicable-for-transform (:id transform))]
(-> transform
(assoc-in [:target :indexes] (mapv :structured requests))
(assoc-in [:target :index-request-ids] (into [] (keep :id) requests))
(assoc :full-incremental-run? (transforms-base.u/full-incremental-run? transform)))))
-(defn execute!
+(mu/defn execute! :- :nil
"Run `transform` and sync its target table.
Executes synchronously, but the start is observable: once the run row is booked in the database,
`:on-start` is called with the transform run id and `:start-promise` (if any) is delivered
`[:started run-id]`. A throw before that point delivers the throwable to the promise instead, so
callers awaiting the start never hang."
- ([transform]
+ ([transform :- ::transforms.schema/transform]
(execute! transform nil))
- ([transform {:keys [start-promise on-start] :as opts}]
+ ([transform :- ::transforms.schema/transform
+ {:keys [start-promise on-start] :as opts} :- [:maybe ::transforms.schema/execute-options]]
(let [opts (merge opts
{:on-start (fn [run-id]
(when start-promise
@@ -48,4 +52,5 @@
;; so a caller awaiting the start isn't left hanging on a pre-start failure;
;; a no-op when the run had already started and the promise was delivered
(when start-promise (deliver start-promise t))
- (throw t))))))
+ (throw t)))
+ nil)))
diff --git a/src/metabase/transforms/feature_gating.clj b/src/metabase/transforms/feature_gating.clj
index 647a543d537..cbcdb901668 100644
--- a/src/metabase/transforms/feature_gating.clj
+++ b/src/metabase/transforms/feature_gating.clj
@@ -1,16 +1,17 @@
(ns metabase.transforms.feature-gating
(:require
[metabase.api.common :as api]
- [metabase.premium-features.core :as premium-features]))
+ [metabase.premium-features.core :as premium-features]
+ [metabase.util.malli :as mu]))
-(defn enabled-source-types
+(mu/defn enabled-source-types :- [:set :string]
"Returns set of enabled source types for WHERE clause filtering."
[]
(cond-> #{}
(premium-features/query-transforms-enabled?) (into ["native" "mbql"])
(premium-features/python-transforms-enabled?) (conj "python")))
-(defn enabled-source-types-for-user
+(mu/defn enabled-source-types-for-user :- [:maybe [:set :string]]
"Returns set of enabled source types for WHERE clause filtering."
[]
(when (api/entitled-data-analyst?)
diff --git a/src/metabase/transforms/freshness.clj b/src/metabase/transforms/freshness.clj
index 345c73e6263..414c0b6471f 100644
--- a/src/metabase/transforms/freshness.clj
+++ b/src/metabase/transforms/freshness.clj
@@ -5,19 +5,24 @@
a slow schedule and so should not be treated as inactive ([[schedule-fresh-transform-ids]])."
(:require
[java-time.api :as t]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.models.transform-schedule :as transform-schedule]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms])
(:import
(java.util Date)
(org.quartz CronExpression)))
(set! *warn-on-reflection* true)
-(defn- fired-since?
+(mu/defn- fired-since? :- [:maybe :boolean]
"True if `cron` has a fire time after `last-success` and at or before `now` — i.e. a scheduled run
was due since the last success. Unparseable crons are ignored."
- [cron ^Date last-success ^Date now]
+ [cron :- :string
+ ^Date last-success :- (ms/InstanceOfClass Date)
+ ^Date now :- (ms/InstanceOfClass Date)]
(try
(when-let [next-fire (.getNextValidTimeAfter (CronExpression. ^String cron) last-success)]
(not (.after next-fire now)))
@@ -25,11 +30,14 @@
(log/warnf "Ignoring unparseable transform job schedule %s: %s" (pr-str cron) (ex-message e))
false)))
-(defn- fresh-ids
+(mu/defn- fresh-ids :- [:set ::lib.schema.id/transform]
"Subset of `ids` that have a timestamp in `id->last-run` and whose schedules have not fired
based on `schedules-by-id` since the `now` timestamp.
IDs with no schedules are fresh once they have a timestamp; IDs with no timestamp are never fresh."
- [now ids schedules-by-id id->last-run]
+ [now :- ms/TemporalInstant
+ ids :- [:maybe [:or [:set ::lib.schema.id/transform] [:sequential ::lib.schema.id/transform]]]
+ schedules-by-id :- [:map-of ::lib.schema.id/transform [:set :string]]
+ id->last-run :- [:maybe [:map-of ::lib.schema.id/transform ms/TemporalInstant]]]
(let [now-date (Date/from (t/instant now))]
(into #{}
(for [id ids
@@ -39,22 +47,23 @@
:when (not-any? #(fired-since? % ran-date now-date) (schedules-by-id id))]
id))))
-(defn schedule-fresh-transform-ids
+(mu/defn schedule-fresh-transform-ids :- [:set ::lib.schema.id/transform]
"IDs of transforms that are between fires of an active job schedule as of `now`.
Covered transforms have ≥1 active scheduled job and no schedule fire time has elapsed since
their most recent run (any status) - e.g. a six-month cadence whose last run was two months
ago - and should not be treated as inactive by recency checks like staleness. Transforms that
have never run are never schedule-fresh."
- [now]
+ [now :- ms/TemporalInstant]
(let [schedules-by-id (transform-schedule/schedules-for-transforms)
ids (keys schedules-by-id)]
(fresh-ids now ids schedules-by-id (transform-run/last-run-start-times ids))))
-(defn fresh-dep-ids
+(mu/defn fresh-dep-ids :- [:maybe [:set ::lib.schema.id/transform]]
"Subset of `dep-ids` safe to skip as of `now`: a dep that has never succeeded is never fresh; a
scheduled one is fresh while none of its schedules has fired since its last success; an unscheduled
one is fresh once it has succeeded."
- [now dep-ids]
+ [now :- ms/TemporalInstant
+ dep-ids :- [:maybe [:or [:set ::lib.schema.id/transform] [:sequential ::lib.schema.id/transform]]]]
(when (seq dep-ids)
(fresh-ids now dep-ids
(transform-schedule/schedules-for-transforms dep-ids)
diff --git a/src/metabase/transforms/instrumentation.clj b/src/metabase/transforms/instrumentation.clj
index e31c8dffbf1..b21c7622127 100644
--- a/src/metabase/transforms/instrumentation.clj
+++ b/src/metabase/transforms/instrumentation.clj
@@ -4,7 +4,8 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
@@ -118,28 +119,36 @@
(assoc labels :type "processed")
rows-processed)))
-(defn record-job-start!
+(mu/defn record-job-start! :- :nil
"Record the start of a transform job run."
- [job-id run-method]
+ [job-id :- ms/PositiveInt
+ run-method :- [:or :keyword :string]]
(log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
(analytics/inc! :metabase-transforms/job-runs-total
- {:run-method (name run-method)}))
+ {:run-method (name run-method)})
+ nil)
-(defn record-job-completion!
+(mu/defn record-job-completion! :- :nil
"Record the successful completion of a transform job run."
- [job-id run-method duration-ms]
+ [job-id :- ms/PositiveInt
+ run-method :- [:or :keyword :string]
+ duration-ms :- :int]
(log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
(let [labels {:run-method (name run-method)}]
(analytics/inc! :metabase-transforms/job-runs-completed labels)
- (analytics/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+ (analytics/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))
+ nil)
-(defn record-job-failure!
+(mu/defn record-job-failure! :- :nil
"Record the failure of a transform job run."
- [job-id run-method duration-ms]
+ [job-id :- ms/PositiveInt
+ run-method :- [:or :keyword :string]
+ duration-ms :- :int]
(log/warnf "Transform job failed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
(let [labels {:run-method (name run-method)}]
(analytics/inc! :metabase-transforms/job-runs-failed labels)
- (analytics/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+ (analytics/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))
+ nil)
(defmacro with-job-timing
"Execute body while timing a transform job run. Automatically records start, completion or failure.
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index fc8362ed3cb..d19f09ebd4a 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -5,6 +5,7 @@
[flatland.ordered.set :as ordered-set]
[metabase.channel.urls :as urls]
[metabase.events.core :as events]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.revisions.core :as revisions]
[metabase.run-tracking.core :as rt]
[metabase.task.core :as task]
@@ -20,12 +21,16 @@
[metabase.transforms.models.job-run :as transforms.job-run]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.models.transform-schedule :as transform-schedule]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.settings :as transforms.settings]
[metabase.transforms.usage :as transforms.usage]
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
@@ -38,12 +43,95 @@
own timeout."
(* transform-job-heartbeat-stale-minutes 60 1000))
-(defn- next-transform [ordering transforms-by-id complete]
+(mr/def ::ids
+ "A set of transform IDs."
+ [:set ::lib.schema.id/transform])
+
+(mr/def ::ordering
+ "Transform IDs to the IDs of the transforms they depend on."
+ [:map-of ::lib.schema.id/transform ::ids])
+
+(mr/def ::transforms-by-id
+ "Transforms keyed by ID."
+ [:map-of ::lib.schema.id/transform ::transforms.schema/transform])
+
+(mr/def ::run-method
+ "How a run was started."
+ [:enum :manual :cron])
+
+(mr/def ::failure
+ "A transform that did not complete in a coordinated run."
+ [:map {:closed true}
+ [::transform ::transforms.schema/transform]
+ [::message :string]
+ [::cascade? {:optional true} :boolean]
+ [::throwable {:optional true} (ms/InstanceOfClass Throwable)]])
+
+(mr/def ::target-key
+ "A coarse identity of the table a transform writes."
+ [:tuple [:maybe :int] [:maybe :string] [:maybe :string]])
+
+(mr/def ::worker
+ "An in-flight transform worker."
+ [:map {:closed true}
+ [:future (ms/InstanceOfClass java.util.concurrent.Future)]
+ [:timer number?]
+ [:tkey [:maybe ::target-key]]
+ [:transform ::transforms.schema/transform]
+ [:started-run-id ::transforms.schema/promise]])
+
+(mr/def ::state
+ "The state of the coordinator loop."
+ [:map {:closed true}
+ [:succeeded ::ids]
+ [:failed ::ids]
+ [:failures [:sequential ::failure]]
+ [:in-flight [:map {:closed true}
+ [:sql [:map-of ::lib.schema.id/transform ::worker]]
+ [:py [:map-of ::lib.schema.id/transform ::worker]]]]
+ [:in-flight-targets [:set ::target-key]]
+ [:aborted? {:optional true} :boolean]])
+
+(mr/def ::context
+ "The coordinator context threaded through the workers."
+ [:map {:closed true}
+ [:plan [:sequential ::transforms.schema/transform]]
+ [:deps ::ordering]
+ [:parent-run [:tuple [:enum :job :dag] ms/PositiveInt]]
+ [:run-gone? [:=> [:cat] :boolean]]
+ [:run-method [:maybe ::run-method]]
+ [:user-id [:maybe ::lib.schema.id/user]]
+ [:add-run-activity! [:=> [:cat] [:maybe :int]]]
+ [:lanes [:map {:closed true} [:sql ms/PositiveInt] [:py ms/PositiveInt]]]
+ [:timeout-ms number?]])
+
+(mr/def ::run-opts
+ "The options of a coordinated run."
+ [:map {:closed true}
+ [:run-method {:optional true} [:maybe ::run-method]]
+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:skip-fresh-deps? {:optional true} :boolean]
+ [:parent-run-type {:optional true} [:enum :job :dag]]
+ [:active-runs-atom {:optional true} :metabase.transforms-base.schema/atom]
+ [:add-run-activity! {:optional true} [:=> [:cat] [:maybe :int]]]])
+
+(mr/def ::run-result
+ "The outcome of a coordinated run."
+ [:map {:closed true}
+ [::status [:enum :succeeded :failed :aborted]]
+ [::failures {:optional true} [:sequential ::failure]]])
+
+(mu/defn- next-transform :- [:maybe ::transforms.schema/transform]
+ [ordering :- ::ordering
+ transforms-by-id :- ::transforms-by-id
+ complete :- ::ids]
(-> (transforms-base.ordering/available-transforms ordering #{} complete)
first
transforms-by-id))
-(defn- sorted-ordering [ordering transforms-by-id]
+(mu/defn- sorted-ordering :- ::ordering
+ [ordering :- ::ordering
+ transforms-by-id :- ::transforms-by-id]
(let [get-created-at (comp :created_at transforms-by-id)]
(into (sorted-map-by (fn [transform-id-1 transform-id-2]
;; compare based on created_at and id. created_at should generally be unique, so id should
@@ -55,7 +143,7 @@
transform-id-2])))
ordering)))
-(defn dependencies->plan
+(mu/defn dependencies->plan :- ::transforms.schema/transform-plan
"Build an execution plan from a `dependencies` map (transform-id -> #{dep-ids}, as returned by
`transform-ordering`) and the transforms it covers. Returns `{:order [...transforms in execution
order] :deps dependencies}`. Throws if the dependency graph contains a cycle.
@@ -63,7 +151,8 @@
This is the pure, in-memory part of planning — no DB or `table-dependencies` work — so callers
that already hold a dependency map (e.g. a DAG reprocess that computed the whole graph once) can
build a plan without recomputing it."
- [dependencies all-transforms]
+ [dependencies :- ::ordering
+ all-transforms :- [:sequential ::transforms.schema/transform]]
(let [transforms-by-id (into {}
(keep (fn [{:keys [id] :as transform}]
(when (contains? dependencies id)
@@ -81,7 +170,8 @@
{:order (map transforms-by-id complete)
:deps dependencies}))))
-(defn- get-plan [transform-ids]
+(mu/defn- get-plan :- ::transforms.schema/transform-plan
+ [transform-ids :- ::ids]
(tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}
(let [all-transforms (transforms.db/transform-dependency-rows)
;; Walk only the dependency closure of the transforms we're asked to run.
@@ -105,27 +195,31 @@
[])]
(dependencies->plan dependencies closure-transforms)))))
-(defn- wait-for-transform-slot!
+(mu/defn- wait-for-transform-slot! :- :boolean
"Poll until no active run exists for `transform-id`, up to `timeout-ms`."
- [transform-id timer timeout-ms]
+ [transform-id :- ::lib.schema.id/transform
+ timer :- number?
+ timeout-ms :- number?]
(loop []
(cond
(not (transform-run/running-run-for-transform-id transform-id)) true
(>= (u/since-ms timer) timeout-ms) false
:else (do (Thread/sleep 2000) (recur)))))
-(defn- run-transform!
+(mu/defn- run-transform! :- [:maybe :int]
"Execute a single transform as part of a coordinated run. `ctx` is the coordinator context; this
uses its `:parent-run` (a `[type id]` tuple — `[:job id]`/`[:dag id]` — that links the created
`transform_run` row to its coordinating `transform_job_run`/`transform_dag_run`), `:run-method`,
`:user-id`, and `:add-run-activity!` (a zero-arg fn called after successful execution to touch the
coordinating run's updated_at)."
- [{:keys [parent-run run-method user-id add-run-activity!]} started-run-id {transform-id :id :as transform}]
+ [{:keys [parent-run run-method user-id add-run-activity!]} :- ::context
+ started-run-id :- ::transforms.schema/promise
+ {transform-id :id :as transform} :- ::transforms.schema/transform]
(cond
- (not (transforms.u/check-feature-enabled transform))
+ (not (transforms.u/check-feature-enabled (transforms.u/->definition transform)))
(log/warnf "Skip running transform %d due to lacking premium features" transform-id)
- (transforms.usage/transform-locked? transform)
+ (transforms.usage/transform-locked? (transforms.u/->definition transform))
(log/warnf "Skip running transform %d due to locked meter (trial quota exhausted)" transform-id)
:else
@@ -158,23 +252,25 @@
(recur))))))
(add-run-activity!))))
-(defn- lane-for
+(mu/defn- lane-for :- [:enum :py :sql]
"Lane a transform runs in: `:py` for python transforms (single-slot), `:sql` otherwise."
- [t]
+ [t :- ::transforms.schema/transform]
(if (transforms-base.u/python-transform? t) :py :sql))
-(defn- busy?
+(mu/defn- busy? :- :boolean
"True while any transform is still in flight in either lane."
- [{:keys [in-flight]}]
- (some seq (vals in-flight)))
+ [{:keys [in-flight]} :- ::state]
+ (boolean (some seq (vals in-flight))))
-(defn- submit-transform!
+(mu/defn- submit-transform! :- (ms/InstanceOfClass java.util.concurrent.Future)
"Run `transform` on its own thread; returns a future yielding a `::status` completion map.
`started-run-id` is a promise the worker delivers its transform run id to once `execute!` has
created it — at most once, since the `:already-running` retry only re-enters after a failed
start — so [[cancel-worker!]] can target exactly that run. `ctx` is the coordinator context,
forwarded to [[run-transform!]] "
- [{[run-type run-id] :parent-run :as ctx} started-run-id transform]
+ [{[run-type run-id] :parent-run :as ctx} :- ::context
+ started-run-id :- ::transforms.schema/promise
+ transform :- ::transforms.schema/transform]
(future
(log/with-context {:run-type run-type, :run-id run-id, :transform-id (:id transform)}
(try
@@ -188,18 +284,19 @@
::message (or (ex-message t) (str t))
::throwable t})))))
-(defn- transform-target-key
+(mu/defn- transform-target-key :- [:maybe ::target-key]
"A coarse identity for a transform's output table, used to keep two transforms that write the
same target table from being dispatched concurrently. Returns nil when the transform has no
resolved target (in which case it isn't subject to the co-writer guard)."
- [transform]
+ [transform :- ::transforms.schema/transform]
(when-let [{:keys [database schema name]} (:target transform)]
[database schema name]))
-(defn- dispatch-ready!
+(mu/defn- dispatch-ready! :- ::state
"Submit every transform whose deps have all succeeded and whose lane has a free slot, recording the
transitive cascade of dep-failures along the way. Returns the updated state."
- [st {:keys [plan deps lanes] :as ctx}]
+ [st :- ::state
+ {:keys [plan deps lanes] :as ctx} :- ::context]
(reduce
(fn [{:keys [succeeded failed in-flight-targets] :as st} t]
(let [id (:id t)
@@ -243,18 +340,19 @@
st
plan))
-(defn- cancel-worker!
+(mu/defn- cancel-worker! :- :boolean
"Stop an in-flight worker: signal cooperative cancelation for the run it started, then interrupt
its thread."
- [{:keys [started-run-id] fut :future}]
+ [{:keys [started-run-id] fut :future} :- ::worker]
(when-let [run-id (deref started-run-id 0 nil)]
(canceling/chan-signal-cancel! run-id))
(future-cancel fut))
-(defn- sweep-workers!
+(mu/defn- sweep-workers! :- ::state
"Fold finished workers into the state; cancel, interrupt, and fail workers that have outrun
`timeout-ms` — the hard backstop for a worker that ignored its own (query/cancel) timeout."
- [st timeout-ms]
+ [st :- ::state
+ timeout-ms :- number?]
(reduce-kv
(fn [st lane-key workers]
(reduce-kv
@@ -286,9 +384,9 @@
st
(:in-flight st)))
-(defn- cancel-in-flight!
+(mu/defn- cancel-in-flight! :- :nil
"Best-effort cancel + interrupt of every in-flight worker."
- [st]
+ [st :- ::state]
(doseq [workers (vals (:in-flight st))
[id worker] workers]
(try
@@ -300,17 +398,18 @@
;; job-run-id -> promise, delivered once the run is found terminated externally (e.g. reaped).
(atom {}))
-(defn- heartbeat-and-reconcile-runs!
+(mu/defn- heartbeat-and-reconcile-runs! :- :nil
"Stamp a heartbeat on every active job run this process owns, then deliver the `gone` promise of
any that were terminated externally so their coordinator aborts."
[]
(coordinated-run/heartbeat-and-reconcile! :model/TransformJobRun active-runs))
-(defn- run-coordinator-loop!
+(mu/defn- run-coordinator-loop! :- ::state
"Dispatch ready transforms; then each round sweep the in-flight workers and refill freed slots —
until nothing is in flight. On an abnormal exit (exception or interrupt) the in-flight workers are
canceled before rethrowing, so they don't keep writing under a run that is about to be failed."
- [init-state {:keys [run-gone? timeout-ms] :as ctx}]
+ [init-state :- ::state
+ {:keys [run-gone? timeout-ms] :as ctx} :- ::context]
(let [last-st (volatile! init-state)
step! (fn [st] (vreset! last-st st) st)]
(try
@@ -334,11 +433,11 @@
(cancel-in-flight! @last-st)
(throw t)))))
-(defn- app-db-now
+(mu/defn- app-db-now :- ms/TemporalInstant
[]
(transforms.db/app-db-now))
-(defn run-transforms!
+(mu/defn run-transforms! :- ::run-result
"Run the transforms of `plan`, honoring the DAG.
Each transform runs on its own thread; SQL transforms run concurrently up to
@@ -356,12 +455,15 @@
Returns `{::status :succeeded}`, `{::status :failed ::failures [...]}`, or `{::status :aborted}`
when the coordinating run was terminated externally (e.g. reaped) while this coordinator was still running."
- [run-id transform-ids-to-run {plan :order deps :deps} {:keys [run-method user-id skip-fresh-deps?
- parent-run-type active-runs-atom add-run-activity!]
- :or {skip-fresh-deps? true
- parent-run-type :job
- active-runs-atom active-runs
- add-run-activity! (constantly nil)}}]
+ [run-id :- ms/PositiveInt
+ transform-ids-to-run :- [:or ::ids [:sequential ::lib.schema.id/transform]]
+ {plan :order deps :deps} :- ::transforms.schema/transform-plan
+ {:keys [run-method user-id skip-fresh-deps?
+ parent-run-type active-runs-atom add-run-activity!]
+ :or {skip-fresh-deps? true
+ parent-run-type :job
+ active-runs-atom active-runs
+ add-run-activity! (constantly nil)}} :- ::run-opts]
(let [gone (promise)
_ (swap! active-runs-atom assoc run-id gone)
final-state
@@ -402,19 +504,24 @@
(seq (:failures final-state)) {::status :failed ::failures (:failures final-state)}
:else {::status :succeeded})))
-(defn- job-transform-ids [job-id]
+(mu/defn- job-transform-ids :- ::ids
+ [job-id :- ms/PositiveInt]
(let [tag-ids (transforms.db/job-tag-ids job-id)]
(if (seq tag-ids)
(or (transforms.db/transform-ids-with-tags tag-ids)
#{})
#{})))
-(defn job-transforms
+(mu/defn job-transforms :- [:sequential [:merge
+ ::transforms.schema/transform
+ [:map {:closed true}
+ [:dependency {:optional true} :boolean]
+ [:scheduled {:optional true} :boolean]]]]
"Return the transforms that are executed when running the job with ID `job-id`, in execution order.
Transforms pulled into the plan only as dependencies are marked with `:dependency true`
and `:scheduled` (whether any active job's schedule covers them)."
- [job-id]
+ [job-id :- ms/PositiveInt]
(let [tagged (job-transform-ids job-id)
plan (:order (get-plan tagged))
dep-ids (into #{} (comp (map :id) (remove tagged)) plan)
@@ -425,21 +532,21 @@
:scheduled (contains? scheduled id))))
plan)))
-(defn- split-cascade-failures
+(mu/defn- split-cascade-failures :- [:tuple [:sequential ::failure] [:maybe [:sequential ::failure]]]
"Split `failures` into `[root-causes cascades]`, where cascades are dependents that never ran
because an upstream transform failed. If somehow every failure is a cascade, treat them all as
root causes so they aren't silently dropped."
- [failures]
+ [failures :- [:sequential ::failure]]
(let [{cascades true, roots false} (group-by (comp boolean ::cascade?) failures)]
(if (seq roots)
[roots cascades]
[failures nil])))
-(defn- compile-transform-failure-messages
+(mu/defn- compile-transform-failure-messages :- :string
"Render the `::failures` of a coordinated run (as returned by [[run-transforms!]]) into a single
human-readable message: one block per root-cause failure with a link to its run, plus a single
summary line for dependents skipped because an upstream transform failed."
- [failures]
+ [failures :- [:sequential ::failure]]
(let [[roots cascades] (split-cascade-failures failures)
blocks (map (fn [failure]
(format "%s %s:\n%s"
@@ -455,13 +562,18 @@
(str/join ", " (map (comp :name ::transform) cascades))))]
(str/join "\n\n" (cond-> blocks summary (concat [summary])))))
-(defn execute-coordinated-run!
+(mu/defn execute-coordinated-run! :- ::run-result
"Run `transform-ids` per `plan` (as returned by [[get-plan]] or [[dependencies->plan]]) as the
members of the coordinated run `run-id` (a row of `model`) and move the row to its terminal
state, failing it if execution throws (the throwable is rethrown). `label` names the run in
logs; `run-opts` are passed through to [[run-transforms!]]. Returns the [[run-transforms!]]
result so callers can act on failures."
- [model run-id transform-ids plan label run-opts]
+ [model :- ::transforms.schema/coordinated-run-model
+ run-id :- ms/PositiveInt
+ transform-ids :- [:or ::ids [:sequential ::lib.schema.id/transform]]
+ plan :- ::transforms.schema/transform-plan
+ label :- :string
+ run-opts :- ::run-opts]
(try
(let [result (run-transforms! run-id transform-ids plan
(assoc run-opts
@@ -485,12 +597,12 @@
(log/errorf "Error when failing %s: %s" label (ex-message e))))
(throw t))))
-(defn- active-users-to-edit-transform
+(mu/defn- active-users-to-edit-transform :- [:maybe [:sequential [:maybe :metabase.users.schema/user]]]
"Return the users that edited the transform, in reverse chron.
Only returns active users.
And each user is unique.
Returns `nil` if there isn't one."
- [transform-id]
+ [transform-id :- ::lib.schema.id/transform]
(when-some [revisions (seq (revisions/revisions :model/Transform transform-id))]
(let [user-ids (map :user_id revisions)
distinct-user-ids (distinct user-ids)
@@ -499,28 +611,32 @@
;; maintain order
(map by-id distinct-user-ids))))
-(defn- active-admins
+(mu/defn- active-admins :- [:sequential :metabase.users.schema/user]
[]
(transforms.db/active-admins))
-(defn- transform-creator
- [transform]
+(mu/defn- transform-creator :- [:sequential :metabase.users.schema/user]
+ [transform :- ::transforms.schema/transform]
(transforms.db/active-users [(:creator_id transform)]))
-(defn- users-to-notify-of-transform-failure [transform]
+(mu/defn- users-to-notify-of-transform-failure :- [:maybe [:sequential [:maybe :metabase.users.schema/user]]]
+ [transform :- ::transforms.schema/transform]
(or (seq (take 1 (active-users-to-edit-transform (:id transform))))
(seq (transform-creator transform))
(seq (active-admins))))
-(defn- structure-message
+(mu/defn- structure-message :- [:map {:closed true}
+ [:first_line [:maybe :string]]
+ [:details [:vector :string]]]
"Split a message into first line and detail lines for safe template rendering."
- [message]
+ [message :- :string]
(let [[first-line & rest] (str/split message #"\n")]
{:first_line first-line
:details (vec rest)}))
-(defn- notify-transform-failures
- [job-id failures]
+(mu/defn- notify-transform-failures :- :nil
+ [job-id :- ms/PositiveInt
+ failures :- [:sequential ::failure]]
;; We intend to notify users of failures during a cron run.
;; The user to notify is the last person to modify the transform
;; or the creator if it hasn't been modified.
@@ -551,13 +667,18 @@
:message (structure-message (::message failure))})
failures)}))))
-(defn run-job!
+(mu/defn run-job! :- [:maybe ms/PositiveInt]
"Runs all transforms for a given job and their dependencies.
`:start-promise`, if provided, is delivered `[:started run-id]` once the run row is created, `nil`
if nothing was run (already running / no transforms), or a Throwable on a pre-start failure.
Returns the run id, or nil if nothing was executed."
- [job-id {:keys [run-method start-promise] :as opts}]
+ [job-id :- ms/PositiveInt
+ {:keys [run-method start-promise] :as opts} :- [:map {:closed true}
+ [:run-method ::run-method]
+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:skip-fresh-deps? {:optional true} :boolean]
+ [:start-promise {:optional true} [:maybe ::transforms.schema/promise]]]]
(try
(if (transforms.job-run/running-run-for-job-id job-id)
(do (log/info "Not executing transform job" (pr-str job-id) "because it is already running")
@@ -600,15 +721,15 @@
(some-> start-promise (deliver t))
(throw t))))
-(defn cancel-job-run!
+(mu/defn cancel-job-run! :- :boolean
"Cancel an in-progress job run and request cancellation of its still-running member transform runs.
Returns true if the run was active and is now canceled, false if it had already finished."
- [run-id]
+ [run-id :- ms/PositiveInt]
(coordinated-run/cancel! :model/TransformJobRun :job_run_id run-id))
;;; ------------------------------------------- Orphan reaping -------------------------------------------
-(defn- reap-orphaned-runs!
+(mu/defn- reap-orphaned-runs! :- [:maybe [:sequential ::transforms.schema/transform-job-run]]
"Reap job runs whose coordinator process died (stale heartbeat)."
[]
(transforms.job-run/reap-orphaned-runs! transform-job-heartbeat-stale-minutes))
diff --git a/src/metabase/transforms/models/dag_run.clj b/src/metabase/transforms/models/dag_run.clj
index f63ef82c60b..a97672725d8 100644
--- a/src/metabase/transforms/models/dag_run.clj
+++ b/src/metabase/transforms/models/dag_run.clj
@@ -5,8 +5,12 @@
job — so it has its own table rather than overloading `transform_job_run`. Member runs link back
via `transform_run.dag_run_id`."
(:require
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.transforms.db :as transforms.db]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -31,13 +35,16 @@
{:status mi/transform-keyword
:direction mi/transform-keyword})
-(defn start-dag-run!
+(mu/defn start-dag-run! :- ::transforms.schema/transform-dag-run
"Start a DAG-reprocess run seeded from `source-transform-id`, traversing `direction`
(`:upstream`/`:downstream`). `user-id` is the user who triggered it; `transform-count` is the
number of transforms the run's plan selected to run, stored for display in run listings.
Snapshots the seed transform's name and entity_id so the run stays displayable after the
transform is deleted (its FK is SET NULL, as for `transform_run.transform_id`)."
- [source-transform-id direction user-id transform-count]
+ [source-transform-id :- ::lib.schema.id/transform
+ direction :- [:enum :upstream :downstream]
+ user-id :- [:maybe ::lib.schema.id/user]
+ transform-count :- :int]
(let [transform (transforms.db/transform-snapshot source-transform-id)]
(transforms.db/insert-dag-run! {:source_transform_id source-transform-id
:source_transform_name (:name transform)
@@ -48,12 +55,12 @@
:status :started
:is_active true})))
-(defn running-run-for-source-transform-id
+(mu/defn running-run-for-source-transform-id :- [:maybe ::transforms.schema/transform-dag-run]
"Return the single active DAG run seeded from `source-transform-id`, or nil."
- [source-transform-id]
+ [source-transform-id :- ::lib.schema.id/transform]
(transforms.db/active-dag-run-for-transform source-transform-id))
-(defn transform-runs-for-dag-run
+(mu/defn transform-runs-for-dag-run :- [:sequential ::transforms.schema/transform-run]
"Return the transform runs that were part of the given DAG run, ordered by start time."
- [dag-run-id]
+ [dag-run-id :- ms/PositiveInt]
(transforms.db/runs-for-dag-run dag-run-id))
diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.clj
index 4883596de59..f5f38c7a60d 100644
--- a/src/metabase/transforms/models/job_run.clj
+++ b/src/metabase/transforms/models/job_run.clj
@@ -4,6 +4,9 @@
[metabase.transforms.coordinated-run :as coordinated-run]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.models.util :as transforms.models.u]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.realize :as t2.realize]))
@@ -29,17 +32,18 @@
{:status mi/transform-keyword
:run_method mi/transform-keyword})
-(defn latest-runs
+(mu/defn latest-runs :- [:maybe [:vector ::transforms.schema/transform-job-run]]
"Return the latest runs for `job-ids`."
- [job-ids]
+ [job-ids :- [:set ms/PositiveInt]]
(when (seq job-ids)
(into [] (map (comp t2.realize/realize #(dissoc % :rn)))
(transforms.db/latest-job-runs-reducible job-ids))))
-(defn start-run!
+(mu/defn start-run! :- ::transforms.schema/transform-job-run
"Start a run. Snapshots the job's name and entity_id so the run stays displayable after the job
is deleted (`transform_job_run.job_id` has no FK; job runs outlive their job)."
- ([job-id run-method]
+ ([job-id :- ms/PositiveInt
+ run-method :- [:enum :manual :cron]]
;; :built_in_type so the after-select hook localizes built-in job names; str realizes the
;; LocalizedString into the snapshot
(let [job (transforms.db/job-snapshot job-id)]
@@ -50,22 +54,35 @@
:status :started
:is_active true}))))
-(defn reap-orphaned-runs!
+(mu/defn reap-orphaned-runs! :- [:maybe [:sequential ::transforms.schema/transform-job-run]]
"Time out active job runs whose `last_heartbeat` is older than `stale-minutes` (their coordinator
process is presumed dead). Returns the rows that were timed out so callers can notify."
- [stale-minutes]
+ [stale-minutes :- ms/PositiveInt]
(coordinated-run/reap-orphaned-runs! :model/TransformJobRun "job" stale-minutes))
-(defn running-run-for-job-id
+(mu/defn running-run-for-job-id :- [:maybe ::transforms.schema/transform-job-run]
"Return a single active job run or nil."
- [id]
+ [id :- ms/PositiveInt]
(transforms.db/active-job-run-for-job id))
-(defn paged-job-runs
+(mu/defn paged-job-runs :- [:map {:closed true}
+ [:data [:sequential ::transforms.schema/transform-job-run]]
+ [:limit ms/PositiveInt]
+ [:offset ms/IntGreaterThanOrEqualToZero]
+ [:total ms/IntGreaterThanOrEqualToZero]]
"Return a page of the list of job runs.
Follows the conventions used by the FE."
- [{:keys [sort-column sort-direction job-id status run-method start-time offset limit]}]
+ [{:keys [sort-column sort-direction job-id status run-method start-time offset limit]}
+ :- [:map {:closed true}
+ [:sort-column {:optional true} [:maybe [:or :keyword :string]]]
+ [:sort-direction {:optional true} [:maybe [:or :keyword :string]]]
+ [:job-id {:optional true} [:maybe ms/PositiveInt]]
+ [:status {:optional true} [:maybe :string]]
+ [:run-method {:optional true} [:maybe :string]]
+ [:start-time {:optional true} [:maybe :string]]
+ [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
+ [:limit {:optional true} [:maybe ms/PositiveInt]]]]
(let [offset (or offset 0)
limit (or limit 20)
[start-at end-at] (when start-time (transforms.models.u/timestamp-range start-time))]
@@ -74,7 +91,7 @@
:offset offset
:total (transforms.db/job-run-count job-id status run-method start-at end-at)}))
-(defn transform-runs-for-job-run
+(mu/defn transform-runs-for-job-run :- [:sequential ::transforms.schema/transform-run]
"Return transform runs that were part of the given job run, ordered by start time."
- [job-run-id]
+ [job-run-id :- ms/PositiveInt]
(transforms.db/runs-for-job-run job-run-id))
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 68db91dd0f1..c04a17bd260 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -8,6 +8,7 @@
[metabase.indexes.models.table-index :as table-index]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
@@ -21,11 +22,14 @@
[metabase.transforms.db :as transforms.db]
[metabase.transforms.freshness :as freshness]
[metabase.transforms.models.transform-run :as transform-run]
- [metabase.transforms.schema]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.instance :as t2.instance]))
@@ -37,39 +41,48 @@
(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped? :hook/dependencies]]
(derive :model/Transform trait))
-(defn- transform-readable?
- "Whether the current user can read `instance`. Any extra `args` (an optional `models-cache`) are
- passed through to `transforms.u/source-tables-readable?`."
- [instance & args]
- (and (transforms.u/check-feature-enabled instance)
- (or api/*is-superuser?*
- (and (api/entitled-data-analyst?)
- (apply transforms.u/source-tables-readable? instance args)))))
-
-(defn- native-transform-write-allowed?
- [instance source-db-id]
+(mr/def ::instance
+ "A Transform as the permission checks see it: a stored row, a hydrated one, one with pending changes merged in, or
+ a create request."
+ [:or
+ ::transforms.schema/transform
+ ::transforms.schema/transform.create])
+
+(mu/defn- transform-readable? :- :boolean
+ "Whether the current user can read `instance`, with its source looked up in `models-cache` when given."
+ ([instance :- ::instance]
+ (transform-readable? instance nil))
+ ([instance :- ::instance
+ models-cache :- [:maybe ::transforms.u/source-models]]
+ (transforms.u/source-readable? (transforms.u/->definition instance) models-cache)))
+
+(mu/defn- native-transform-write-allowed? :- [:maybe :boolean]
+ [instance :- ::instance
+ source-db-id :- [:maybe ::lib.schema.id/database]]
(or (not (transforms-base.u/native-query-transform? instance))
(and source-db-id
(= :query-builder-and-native
(perms/full-database-permission-for-user api/*current-user-id* :perms/create-queries source-db-id)))))
-(defn- transform-database-permissions?
- [instance]
+(mu/defn- transform-database-permissions? :- :boolean
+ [instance :- ::instance]
(let [source-db-id (or (transforms-base.i/source-db-id instance) (:source_database_id instance))
target-db-id (transforms-base.i/target-db-id instance)]
(every? #(perms/has-db-transforms-permission? api/*current-user-id* %)
(distinct (keep identity [source-db-id target-db-id])))))
-(defn- transform-writable?
- "Whether the current user can write `instance`. Any extra `args` (an optional `models-cache`) are
- passed through to the source-readability check, as in `transform-readable?`."
- [instance & args]
- (and (remote-sync/transforms-editable?)
- (transforms.u/check-feature-enabled instance)
- (or api/*is-superuser?*
- (and (apply transform-readable? instance args)
- (transform-database-permissions? instance)
- (native-transform-write-allowed? instance (:source_database_id instance))))))
+(mu/defn- transform-writable? :- [:maybe :boolean]
+ "Whether the current user can write `instance`, with its source looked up in `models-cache` when given."
+ ([instance :- ::instance]
+ (transform-writable? instance nil))
+ ([instance :- ::instance
+ models-cache :- [:maybe ::transforms.u/source-models]]
+ (and (remote-sync/transforms-editable?)
+ (transforms.u/check-feature-enabled (transforms.u/->definition instance))
+ (or api/*is-superuser?*
+ (and (transform-readable? instance models-cache)
+ (transform-database-permissions? instance)
+ (native-transform-write-allowed? instance (:source_database_id instance)))))))
(defmethod mi/can-read? :model/Transform
([instance]
@@ -92,52 +105,35 @@
([model pk]
(mi/can-read? model pk)))
+(defmethod mi/can-update? :model/Transform
+ [instance changes]
+ (and (mi/can-write? instance)
+ (mi/can-write? (merge instance changes))))
+
(defmethod mi/can-create? :model/Transform
[_model instance]
;; Inline can-write? logic since instance is a plain map without model metadata.
;; can-write? requires: can-read?, has-db-transforms-permission?, and transforms-editable?
;; can-read? requires: is-superuser? OR (entitled-data-analyst? AND source-tables-readable?)
(and (remote-sync/transforms-editable?)
- (transforms.u/check-feature-enabled instance)
+ (transforms.u/check-feature-enabled (transforms.u/->definition instance))
(or api/*is-superuser?*
(let [source-db-id (or (:source_database_id instance) (transforms-base.i/source-db-id instance))]
(and (api/entitled-data-analyst?)
- (transforms.u/source-tables-readable? instance)
+ (transforms.u/source-tables-readable? (transforms.u/->definition instance))
(transform-database-permissions? instance)
(native-transform-write-allowed? instance source-db-id))))))
-(defn- orphan-query?
- "True when the query map has its `:database` key explicitly set to nil — the
- signature of a transform body whose source database has been deleted (e.g.
- imported from a serdes export of an orphan). We preserve such bodies verbatim
- rather than running them through MBQL normalization, which requires a database."
- [q]
- (or (and (contains? q :database) (nil? (:database q)))
- (and (contains? q "database") (nil? (get q "database")))))
-
-(defn transform-source-out
- "Deserialize a transform source map from JSON storage format.
- Normalizes queries and keywordizes type fields."
- [m]
- (-> m
- mi/json-out-without-keywordization
- (update-keys keyword)
- (m/update-existing :query (fn [q] (if (orphan-query? q) q (lib-be/normalize-query q))))
- (m/update-existing :source-incremental-strategy
- (fn [strategy]
- (-> (update-keys strategy keyword)
- (m/update-existing :lookback #(some-> % (update-keys keyword))))))
- (m/update-existing :source-tables (fn [st] (mapv #(update-keys % keyword) st)))
- (m/update-existing :type keyword)))
-
-(defn transform-source-in
+(mu/defn transform-source-out :- ::transforms.schema/transform-source
+ "Deserialize a transform source map from JSON storage format, normalized to its schema."
+ [m :- [:or :string [:fn map?]]]
+ (lib/normalize ::transforms.schema/transform-source (cond-> m (string? m) mi/json-out-without-keywordization)))
+
+(mu/defn transform-source-in :- :string
"Serialize a transform source map for JSON storage."
- [m]
+ [m :- ::transforms.schema/transform-source]
(-> m
- (m/update-existing :query (fn [q]
- (if (orphan-query? q)
- q
- ((comp lib/prepare-for-serialization lib-be/normalize-query) q))))
+ (m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
mi/json-in))
(t2/deftransforms :model/Transform
@@ -153,32 +149,30 @@
#{:transforms})
(t2/define-before-insert :model/Transform
- [{:keys [source collection_id source_database_id] :as transform}]
- (collection/check-collection-namespace :model/Transform collection_id)
- (when collection_id
- (collection/check-allowed-content :model/Transform collection_id))
- (let [target-db-id (transforms-base.i/target-db-id transform)
- valid-db-id? (and target-db-id (transforms.db/database-exists? target-db-id))]
- ;; Don't warn when target-db-id is nil — that's an orphan source (e.g. a
- ;; serdes-imported transform whose source database is missing), not a
- ;; misconfiguration. Only warn when an id is supplied but invalid.
- (when (and target-db-id (not valid-db-id?))
- (log/warnf "Invalid target database id (%s) ignored for new transform" target-db-id))
- (-> transform
- (assoc-in [:target :database] target-db-id)
- (assoc
- :source_type (transforms-base.u/transform-source-type source)
- :target_db_id (when valid-db-id? target-db-id)
- :source_database_id (or source_database_id (transforms-base.i/source-db-id transform))))))
-
-(defn- resolve-merge-key-field-ids
+ [transform]
+ (let [{:keys [source collection_id source_database_id]} transform]
+ (collection/check-collection-namespace :model/Transform collection_id)
+ (when collection_id
+ (collection/check-allowed-content :model/Transform collection_id))
+ (let [target-db-id (transforms-base.i/target-db-id transform)
+ valid-db-id? (and target-db-id (transforms.db/database-exists? target-db-id))]
+ (when (and target-db-id (not valid-db-id?))
+ (log/warnf "Invalid target database id (%s) ignored for new transform" target-db-id))
+ (-> transform
+ (assoc-in [:target :database] target-db-id)
+ (assoc
+ :source_type (transforms-base.u/transform-source-type source)
+ :target_db_id (when valid-db-id? target-db-id)
+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))))))
+
+(mu/defn- resolve-merge-key-field-ids :- ::transforms.schema/transform
"Fill `:field-id` on a merge target's unique-key columns from the synced target table's fields, once
the table exists and any are still by-name only."
- [transform]
+ [transform :- ::transforms.schema/transform]
(let [target (or (:target transform) (:target (t2/original transform)))
table-id (or (:target_table_id transform) (:target_table_id (t2/original transform)))
unique-key (get-in target [:target-incremental-strategy :unique-key])]
- (if (and (transforms-base.u/merge-target? {:target target})
+ (if (and (transforms-base.u/merge-target? (assoc transform :target target))
table-id
(some #(nil? (:field-id %)) unique-key))
(let [name->id (transforms.db/active-field-ids-by-name table-id)]
@@ -189,41 +183,42 @@
transform)))
(t2/define-before-update :model/Transform
- [{:keys [source source_database_id] :as transform}]
- (when-let [new-collection (:collection_id (t2/changes transform))]
- (collection/check-collection-namespace :model/Transform new-collection)
- (collection/check-allowed-content :model/Transform new-collection))
- ;; The target db is recomputed when source changes because for MBQL transforms,
- ;; the source query's :database is the source of truth for the target database.
- (let [target-changed? (or (:source (t2/changes transform)) (:target (t2/changes transform)))
- target-db-id (when target-changed?
- ;; No database existence check added here, unlike for insert.
- ;; Just allow updates for an invalid target to fail.
- (transforms-base.i/target-db-id transform))]
- ;; A source or target edit may change the output columns, and we can't cheaply tell, so invalidate conservatively:
- ;; the next run re-applies the managed indexes against the new schema and fails if a column is gone.
- (when (and target-changed? (not mi/*deserializing?*))
- (table-index/mark-for-revalidation! (:id transform)))
- (cond-> transform
- source
- (assoc :source_type (transforms-base.u/transform-source-type source)
- :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
-
- ;; Invalidate cached deps when the source changes
- (:source (t2/changes transform))
- (assoc :table_dependencies nil)
-
- target-changed?
- (assoc :target_db_id target-db-id)
-
- ;; Reset checkpoint when the incremental filter field changes
- (let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
- new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]
- (and old-field-id (not= old-field-id new-field-id)))
- (assoc :last_checkpoint_value nil)
-
- true
- resolve-merge-key-field-ids)))
+ [transform]
+ (let [{:keys [source source_database_id]} transform]
+ (when-let [new-collection (:collection_id (t2/changes transform))]
+ (collection/check-collection-namespace :model/Transform new-collection)
+ (collection/check-allowed-content :model/Transform new-collection))
+ ;; The target db is recomputed when source changes because for MBQL transforms,
+ ;; the source query's :database is the source of truth for the target database.
+ (let [target-changed? (or (:source (t2/changes transform)) (:target (t2/changes transform)))
+ target-db-id (when target-changed?
+ ;; No database existence check added here, unlike for insert.
+ ;; Just allow updates for an invalid target to fail.
+ (transforms-base.i/target-db-id transform))]
+ ;; A source or target edit may change the output columns, and we can't cheaply tell, so invalidate conservatively:
+ ;; the next run re-applies the managed indexes against the new schema and fails if a column is gone.
+ (when (and target-changed? (not mi/*deserializing?*))
+ (table-index/mark-for-revalidation! (:id transform)))
+ (cond-> transform
+ source
+ (assoc :source_type (transforms-base.u/transform-source-type source)
+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
+
+ ;; Invalidate cached deps when the source changes
+ (:source (t2/changes transform))
+ (assoc :table_dependencies nil)
+
+ target-changed?
+ (assoc :target_db_id target-db-id)
+
+ ;; Reset checkpoint when the incremental filter field changes
+ (let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
+ new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]
+ (and old-field-id (not= old-field-id new-field-id)))
+ (assoc :last_checkpoint_value nil)
+
+ true
+ resolve-merge-key-field-ids))))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
@@ -231,11 +226,13 @@
(assoc transform :source_type (transforms-base.u/transform-source-type source))
transform))
-(defn- hydrate-permission
+(mu/defn- hydrate-permission :- [:sequential ::instance]
"Batched-hydrate helper: attach a permission under `k` to each transform by calling `pred`
(`transform-readable?`/`transform-writable?`) with a `models-cache` prefetched once for the whole
list, so checking N transforms doesn't issue a query per transform."
- [k transforms pred]
+ [k :- :keyword
+ transforms :- [:sequential ::instance]
+ pred :- [:=> [:cat ::instance ::transforms.u/source-models] [:maybe :boolean]]]
(let [models-cache (transforms.u/prefetch-source-models transforms)]
(mi/instances-with-hydrated-data
transforms k
@@ -366,10 +363,11 @@
(search.core/delete! :model/Transform [(str (:id transform))])
transform)
-(defn update-transform-tags!
+(mu/defn update-transform-tags! :- :nil
"Update the tags associated with a transform using smart diff logic. Only modifies what has changed: deletes removed
tags, updates positions for moved tags, and inserts new tags. Duplicate tag IDs are automatically deduplicated."
- [transform-id tag-ids]
+ [transform-id :- [:maybe ::lib.schema.id/transform]
+ tag-ids :- [:sequential ms/PositiveInt]]
(when transform-id
(t2/with-transaction [_conn]
(let [;; Deduplicate while preserving order of first occurrence
@@ -403,7 +401,8 @@
(transforms.db/insert-transform-tag-links! (for [tag-id to-insert]
{:transform_id transform-id
:tag_id tag-id
- :position (get new-positions tag-id)})))))))
+ :position (get new-positions tag-id)})))))
+ nil))
;;; ------------------------------------------------- Serialization ------------------------------------------------
@@ -442,36 +441,30 @@
(assoc transform :table
(get id->table (:target_table_id transform))))))
-(defn- import-maybe-int-database-fk
+(mu/defn- import-maybe-int-database-fk :- [:maybe ms/PositiveInt]
"Import a database reference back to an ID. Tolerates raw numeric IDs from older exports
where source-tables database_id values were serialized without conversion."
- [v]
+ [v :- [:maybe [:or ms/PositiveInt :string]]]
(if (pos-int? v) v (serdes/*import-database-fk* v)))
-(defn- import-maybe-int-table-fk
+(mu/defn- import-maybe-int-table-fk :- [:maybe ms/PositiveInt]
"Import a table reference back to an ID. Tolerates raw numeric IDs from older exports
where source-tables table_id values were serialized without conversion."
- [v]
+ [v :- [:maybe [:or ms/PositiveInt [:tuple :string [:maybe :string] :string]]]]
(if (pos-int? v) v (serdes/*import-table-fk* v)))
-(defn- import-maybe-int-field-fk [v]
+(mu/defn- import-maybe-int-field-fk :- [:maybe ms/PositiveInt]
+ [v :- [:maybe [:or ms/PositiveInt [:cat :string [:maybe :string] :string [:+ :string]]]]]
(if (pos-int? v) v (some-> v serdes/*import-field-fk*)))
-(defn- export-checkpoint-field-fk
+(mu/defn- export-checkpoint-field-fk :- [:or ::lib.schema.id/field [:cat :string [:maybe :string] :string [:+ :string]]]
"Portable ref for the checkpoint field. A dangling id (field since deleted) is exported as-is:
the importer passes numbers through, so one stale config can't fail the whole import."
- [field-id]
+ [field-id :- ::lib.schema.id/field]
(if (transforms.db/field-exists? field-id)
(serdes/*export-field-fk* field-id)
field-id))
-(defn- update-checkpoint-field
- "Apply `f` to the source's `:checkpoint-filter-field-id`, when one is set."
- [source f]
- (m/update-existing source :source-incremental-strategy
- (fn [strategy]
- (m/update-existing strategy :checkpoint-filter-field-id f))))
-
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
@@ -480,47 +473,36 @@
:owner_user_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
:source_database_id (serdes/fk :model/Database)
- :source {:export-with-context
- (fn [{:keys [source_database_id]} _k source]
- (if source_database_id
- (-> source
- (m/update-existing :query serdes/export-mbql)
- (m/update-existing :source-database serdes/*export-database-fk*)
- (m/update-existing :source-tables
- (fn [entries]
- (->> (transforms-base.u/normalize-source-tables entries)
- (mapv #(-> %
- (m/update-existing :table_id serdes/*export-table-fk*)
- (m/update-existing :database_id serdes/*export-database-fk*))))))
- (update-checkpoint-field export-checkpoint-field-fk))
- ;; Orphan: source DB has been deleted, so table/field rows it referenced
- ;; are gone too. Null the dead numeric refs and flag the body so
- ;; the importer skips ref resolution.
- (-> source
- (assoc :serdes/unresolved true)
- (m/update-existing :query #(-> % (dissoc :lib/metadata) (assoc :database nil)))
- (m/update-existing :source-database (constantly nil))
- (m/update-existing :source-tables
- #(mapv (fn [e] (assoc e :table_id nil :database_id nil)) %))
- (update-checkpoint-field (constantly nil)))))
+ :source {:export
+ (fn [source]
+ (-> source
+ (m/update-existing :query serdes/export-mbql)
+ (m/update-existing :source-database serdes/*export-database-fk*)
+ (m/update-existing :source-tables
+ (fn [entries]
+ (->> (transforms-base.u/normalize-source-tables entries)
+ (mapv #(-> %
+ (m/update-existing :table_id serdes/*export-table-fk*)
+ (m/update-existing :database_id serdes/*export-database-fk*))))))
+ (m/update-existing-in [:source-incremental-strategy :checkpoint-filter-field-id]
+ export-checkpoint-field-fk)))
:import
(fn [source]
- (if (:serdes/unresolved source)
- (-> source
- (dissoc :serdes/unresolved)
- (m/update-existing :query dissoc :lib/metadata))
- (-> source
- (m/update-existing :query serdes/import-mbql)
- (m/update-existing :source-database import-maybe-int-database-fk)
- (m/update-existing :source-tables
- (fn [entries]
- (let [entries (cond-> entries (map? entries) (update-keys name))]
- (->> (cond-> entries (map? entries) transforms-base.u/source-tables-map->vec)
- (mapv (fn [entry]
- (-> entry
- (m/update-existing :table_id import-maybe-int-table-fk)
- (m/update-existing :database_id import-maybe-int-database-fk))))))))
- (update-checkpoint-field import-maybe-int-field-fk))))}
+ (lib/normalize
+ ::transforms.schema/transform-source
+ (-> source
+ (m/update-existing :query serdes/import-mbql)
+ (m/update-existing :source-database import-maybe-int-database-fk)
+ (m/update-existing :source-tables
+ (fn [entries]
+ (let [import-entry #(-> %
+ (m/update-existing :table_id import-maybe-int-table-fk)
+ (m/update-existing :database_id import-maybe-int-database-fk))]
+ (if (map? entries)
+ (update-vals entries #(if (map? %) (import-entry %) (import-maybe-int-table-fk %)))
+ (mapv import-entry entries)))))
+ (m/update-existing-in [:source-incremental-strategy :checkpoint-filter-field-id]
+ import-maybe-int-field-fk))))}
:target {:export #(serdes/export-mbql (dissoc % :table_id))
:import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id (merge {:sort-by (juxt :position :entity_id)} opts))
@@ -534,7 +516,15 @@
[[{:model "Collection" :id collection_id}]])
(for [{tag-id :tag_id} tags]
[{:model "TransformTag" :id tag-id}])
- (serdes/mbql-deps false source))))
+ (when-not (:serdes/unresolved source)
+ (serdes/mbql-deps false source)))))
+
+(defmethod serdes/load-one! "Transform"
+ [ingested maybe-local]
+ (if (get-in ingested [:source :serdes/unresolved])
+ (log/warnf "Skipping transform %s, since an export from before 66 left it without its deleted source database"
+ (:entity_id ingested))
+ (serdes/default-load-one! ingested maybe-local)))
(defmethod serdes/storage-path "Transform" [transform ctx]
(serdes/storage-default-collection-path transform ctx "transforms"))
@@ -544,10 +534,13 @@
(when-let [collection-id (transforms.db/transform-collection-id id)]
{["Collection" collection-id] {"Transform" id}}))
-(defn- maybe-extract-transform-query-text
+(mu/defn- maybe-extract-transform-query-text :- [:maybe :string]
"Return the query text (truncated to `max-searchable-value-length`) from transform source; else nil.
Extracts SQL from query-type transforms and Python code from python-type transforms."
- [{transform-source-type :source_type source :source}]
+ [{transform-source-type :source_type source :source}
+ :- [:map {:closed true}
+ [:source_type [:maybe [:or :keyword :string]]]
+ [:source [:or :string [:fn map?]]]]]
(let [source-data (transform-source-out source)
;; Use the top-level :source_type field since it differentiates between MBQL vs native transforms
query-text (case (keyword transform-source-type)
@@ -557,10 +550,10 @@
(when query-text
(subs query-text 0 (min (count query-text) search.core/max-searchable-value-length)))))
-(defn transforms-with-tags
+(mu/defn transforms-with-tags :- [:sequential ::transforms.schema/transform]
"Returns all transforms associated with the given tag IDs.
Return empty list if no tag IDs are provided or no transforms are associated with the tags."
- [tag-ids]
+ [tag-ids :- [:maybe [:or [:set ms/PositiveInt] [:sequential ms/PositiveInt]]]]
(or (when (seq tag-ids)
(when-let [transform-ids (transforms.db/transform-ids-with-tags tag-ids)]
(transforms.db/transforms transform-ids)))
diff --git a/src/metabase/transforms/models/transform_job.clj b/src/metabase/transforms/models/transform_job.clj
index 50ec32f59b6..5e1cb42ab9e 100644
--- a/src/metabase/transforms/models/transform_job.clj
+++ b/src/metabase/transforms/models/transform_job.clj
@@ -10,9 +10,11 @@
[metabase.transforms.models.job-run :as transforms.job-run]
[metabase.transforms.models.transform :as transform]
[metabase.transforms.schedule :as transforms.schedule]
- [metabase.transforms.schema]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -81,13 +83,13 @@
(for [job jobs]
(assoc job :last_run (get last-executions (:id job)))))))
-(defn- active-flip-lock-name
+(mu/defn- active-flip-lock-name :- :keyword
"Per-job cluster-lock keyword used to serialize `:active` flips of the same job across the
cluster. Concurrent flips of *different* jobs run in parallel."
- [job-id]
+ [job-id :- ms/PositiveInt]
(keyword "metabase.transforms.transform-job-active" (str job-id)))
-(defn activate-job!
+(mu/defn activate-job! :- :nil
"Activate a transform job: set `:active` to true and (re)create its Quartz trigger from the
job's stored schedule so cron firings begin again. Idempotent — calling on an already-active
job is a no-op.
@@ -95,28 +97,31 @@
Concurrent activate/deactivate of the same job is serialized by a per-job cluster lock so
the app-DB row and the Quartz trigger cannot diverge. The lock is held across the trigger
write."
- [job-id]
+ [job-id :- ms/PositiveInt]
(cluster-lock/with-cluster-lock (active-flip-lock-name job-id)
(when (pos? (transforms.db/activate-job! job-id))
- (transforms.schedule/initialize-job! (transforms.db/job job-id)))))
+ (transforms.schedule/initialize-job! (transforms.db/job job-id)))
+ nil))
-(defn deactivate-job!
+(mu/defn deactivate-job! :- :nil
"Deactivate a transform job: set `:active` to false and remove its Quartz trigger so cron
firings stop. Manual runs via the API still work. Idempotent — calling on an already-inactive
job is a no-op.
Concurrent activate/deactivate of the same job is serialized by a per-job cluster lock — see
[[activate-job!]]."
- [job-id]
+ [job-id :- ms/PositiveInt]
(cluster-lock/with-cluster-lock (active-flip-lock-name job-id)
(when (pos? (transforms.db/deactivate-job! job-id))
- (transforms.schedule/delete-trigger! job-id))))
+ (transforms.schedule/delete-trigger! job-id))
+ nil))
-(defn update-job-tags!
+(mu/defn update-job-tags! :- :nil
"Update the tags associated with a job using smart diff logic.
Only modifies what has changed: deletes removed tags, updates positions for moved tags,
and inserts new tags. Duplicate tag IDs are automatically deduplicated."
- [job-id tag-ids]
+ [job-id :- [:maybe ms/PositiveInt]
+ tag-ids :- [:sequential ms/PositiveInt]]
(when job-id
(t2/with-transaction [_conn]
(let [;; Deduplicate, just in case
@@ -150,9 +155,13 @@
(transforms.db/insert-job-tag-links! (for [tag-id to-insert]
{:job_id job-id
:tag_id tag-id
- :position (get new-positions tag-id)})))))))
+ :position (get new-positions tag-id)})))))
+ nil))
-(defn- translated-name-and-description [job]
+(mu/defn- translated-name-and-description :- [:map {:closed true}
+ [:name [:maybe mu/localized-string-schema]]
+ [:description [:maybe mu/localized-string-schema]]]
+ [job :- [:or ::transforms.schema/transform-job ::transforms.schema/transform-job.update]]
(let [values {"hourly"
[(i18n/deferred-trs "Hourly job")
(i18n/deferred-trs "Executes transforms tagged with ''hourly'' every hour")]
diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.clj
index 5ae42270ed4..a8da3f767b2 100644
--- a/src/metabase/transforms/models/transform_run.clj
+++ b/src/metabase/transforms/models/transform_run.clj
@@ -4,15 +4,20 @@
[metabase.api.common :as api]
[metabase.collections.models.collection.root :as collection.root]
[metabase.events.core :as events]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.premium-features.core :as premium-features]
[metabase.run-tracking.core :as rt]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.models.transform-run-cancelation :as cancel]
[metabase.transforms.models.util :as transforms.models.u]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.realize :as t2.realize])
@@ -55,20 +60,46 @@
[transform]
(transforms.db/runs-for-transform (:id transform)))
-(defn latest-runs
+(mr/def ::canceled-run
+ "A TransformRun force-canceled by [[cancel-old-canceling-runs!]], with the time its cancelation was requested."
+ [:merge
+ ::transforms.schema/transform-run
+ [:map {:closed true}
+ [:request_time ms/TemporalInstant]]])
+
+(mr/def ::paged-run-transform
+ "The `:transform` hydrated onto a run listed by [[paged-runs]]: a summary of its live Transform, or the name
+ snapshotted on a run whose Transform was deleted."
+ [:map {:closed true}
+ [:id {:optional true} ::lib.schema.id/transform]
+ [:name {:optional true} :string]
+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]
+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection-or-root]]
+ [:tag_ids {:optional true} [:sequential ms/PositiveInt]]
+ [:deleted {:optional true} :boolean]])
+
+(mr/def ::paged-run
+ "A TransformRun listed by [[paged-runs]], with its `:transform` hydrated."
+ [:merge
+ ::transforms.schema/transform-run
+ [:map {:closed true}
+ [:transform {:optional true} [:maybe ::paged-run-transform]]]])
+
+(mu/defn latest-runs :- [:maybe [:vector ::transforms.schema/transform-run]]
"Return the latest runs for `transform-ids`."
- [transform-ids]
+ [transform-ids :- [:set ::lib.schema.id/transform]]
(when (seq transform-ids)
(into [] (map (comp t2.realize/realize #(dissoc % :rn)))
(transforms.db/latest-runs-reducible transform-ids))))
-(defn start-run!
+(mu/defn start-run! :- ::transforms.schema/transform-run
"Start a run. If `user_id` is provided in properties, it will be stored with the run
and used for attribution in the audit log (avoiding 'External user' for scheduled runs).
Also captures `transform_name` and `transform_entity_id` for historical reference."
- ([transform-id]
+ ([transform-id :- ::lib.schema.id/transform]
(start-run! transform-id {}))
- ([transform-id properties]
+ ([transform-id :- ::lib.schema.id/transform
+ properties :- ::transforms.schema/transform-run.update]
(let [transform (transforms.db/transform-snapshot transform-id)
metered-as (premium-features/transform-metered-as (:source_type transform))
run (transforms.db/insert-run! (assoc properties
@@ -84,53 +115,58 @@
(:user_id run) (assoc :user-id (:user_id run))))
run)))
-(defn succeed-started-run!
+(mu/defn succeed-started-run! :- :int
"Mark a started run as successfully completed."
- ([run-id]
+ ([run-id :- ms/PositiveInt]
(succeed-started-run! run-id {}))
- ([run-id properties]
+ ([run-id :- ms/PositiveInt
+ properties :- ::transforms.schema/transform-run.update]
(u/prog1 (transforms.db/finish-active-run! run-id
(merge properties
{:status :succeeded
:is_active nil}))
(cancel/delete-cancelation! run-id))))
-(defn fail-started-run!
+(mu/defn fail-started-run! :- :int
"Mark the started active run as failed and inactive."
- [run-id properties]
+ [run-id :- ms/PositiveInt
+ properties :- ::transforms.schema/transform-run.update]
(u/prog1 (transforms.db/finish-active-run! run-id
(merge properties
{:status :failed
:is_active nil}))
(cancel/delete-cancelation! run-id)))
-(defn cancel-run!
+(mu/defn cancel-run! :- :int
"Cancel a started run."
- ([run-id]
+ ([run-id :- ms/PositiveInt]
(cancel-run! run-id {:message "Canceled by user"}))
- ([run-id properties]
+ ([run-id :- ms/PositiveInt
+ properties :- ::transforms.schema/transform-run.update]
(u/prog1 (transforms.db/finish-active-run! run-id
(merge properties
{:status :canceled
:is_active nil}))
(cancel/delete-cancelation! run-id))))
-(defn- publish-timeout-event!
+(mu/defn- publish-timeout-event! :- :nil
"Publish `:event/transform-run-timeout` for `run`. Wrapped so that audit-log handler
failures don't bubble into the caller's timeout flow."
- [run]
+ [run :- ::transforms.schema/transform-run]
(try
(events/publish-event! :event/transform-run-timeout
(cond-> {:object run}
(:user_id run) (assoc :user-id (:user_id run))))
(catch Throwable t
- (log/warnf "Failed to publish transform-run-timeout event for run %s: %s" (pr-str (:id run)) (ex-message t)))))
+ (log/warnf "Failed to publish transform-run-timeout event for run %s: %s" (pr-str (:id run)) (ex-message t))))
+ nil)
-(defn timeout-run!
+(mu/defn timeout-run! :- :int
"Mark a started run as timed out."
- ([run-id]
+ ([run-id :- ms/PositiveInt]
(timeout-run! run-id {}))
- ([run-id properties]
+ ([run-id :- ms/PositiveInt
+ properties :- ::transforms.schema/transform-run.update]
(u/prog1 (transforms.db/finish-active-run! run-id
(merge properties
{:message "Timed out"
@@ -142,10 +178,13 @@
(when-let [run (transforms.db/run run-id)]
(publish-timeout-event! run))))))
-(defn- reap-transform-runs!
+(mu/defn- reap-transform-runs! :- [:maybe [:sequential ::transforms.schema/transform-run]]
"Reap active transform runs by `stale-column` into a timeout carrying `message`, publishing a timeout
event per run. See [[metabase.run-tracking.core/reap-orphaned!]]."
- [stale-column age unit message]
+ [stale-column :- [:enum :start_time :last_heartbeat]
+ age :- ms/PositiveInt
+ unit :- :keyword
+ message :- :string]
(let [end-time (OffsetDateTime/now ZoneOffset/UTC)
reaped (rt/reap-orphaned!
{:model :model/TransformRun
@@ -166,24 +205,26 @@
(cancel/delete-old-canceling-runs!)
reaped))
-(defn timeout-old-runs!
+(mu/defn timeout-old-runs! :- [:maybe [:sequential ::transforms.schema/transform-run]]
"Time out all active runs whose `start_time` is older than the specified age. Returns the rows that were
timed out."
- [age unit]
+ [age :- ms/PositiveInt
+ unit :- :keyword]
(reap-transform-runs! :start_time age unit "Timed out by metabase"))
-(defn heartbeat-runs!
+(mu/defn heartbeat-runs! :- :nil
"Stamp `last_heartbeat = now` on the given still-active `run-ids`."
- [run-ids]
- (rt/heartbeat-ids! :model/TransformRun [:is_active true] :last_heartbeat run-ids))
+ [run-ids :- [:sequential ms/PositiveInt]]
+ (rt/heartbeat-ids! :model/TransformRun [:is_active true] :last_heartbeat run-ids)
+ nil)
-(defn reap-orphaned-runs!
+(mu/defn reap-orphaned-runs! :- [:maybe [:sequential ::transforms.schema/transform-run]]
"Time out active runs whose `last_heartbeat` is older than `stale-minutes` (their owning process is
presumed dead). Returns the rows that were timed out."
- [stale-minutes]
+ [stale-minutes :- ms/PositiveInt]
(reap-transform-runs! :last_heartbeat stale-minutes :minute "Timed out: crashed"))
-(defn cancel-old-canceling-runs!
+(mu/defn cancel-old-canceling-runs! :- [:maybe [:vector ::canceled-run]]
"Atomically force-cancels active runs whose cancelation requests are older than `age` `unit`. Returns the
pre-update run rows we transitioned, each augmented with `:request_time` from its cancelation row, so callers
can emit observability only for runs we actually changed.
@@ -192,7 +233,8 @@
concurrent writers (`cancel-run!`, `timeout-run!`) block until we commit and the matching UPDATE-by-id hits
exactly the locked rows. Rows another writer already transitioned (no longer `is_active`) simply drop out of
the lock set and are not reported."
- [age unit]
+ [age :- ms/PositiveInt
+ unit :- :keyword]
(t2/with-transaction [_conn]
(let [times (into {} (map (juxt :run_id :time))
(transforms.db/cancelations-requested-before age unit))
@@ -205,30 +247,30 @@
(when (seq locked)
(transforms.db/runs (mapv :id locked)))))))
-(defn running-run-for-transform-id
+(mu/defn running-run-for-transform-id :- [:maybe ::transforms.schema/transform-run]
"Return a single active transform run or nil."
- [transform-id]
+ [transform-id :- [:maybe ::lib.schema.id/transform]]
(transforms.db/active-run-for-transform transform-id))
-(defn last-successful-run-times
+(mu/defn last-successful-run-times :- [:maybe [:map-of ::lib.schema.id/transform [:maybe ms/TemporalInstant]]]
"Map each id in `transform-ids` with a succeeded run to its most recent run's `end_time`. Ids with
no succeeded run are absent."
- [transform-ids]
+ [transform-ids :- [:set ::lib.schema.id/transform]]
(when (seq transform-ids)
(into {}
(map (juxt :transform_id :last_success))
(transforms.db/last-success-times transform-ids))))
-(defn last-run-start-times
+(mu/defn last-run-start-times :- [:maybe [:map-of ::lib.schema.id/transform [:maybe ms/TemporalInstant]]]
"Map each id in `transform-ids` to its most recent run's `start_time`, regardless of run status.
Ids with no runs are absent."
- [transform-ids]
+ [transform-ids :- [:maybe [:or [:set ::lib.schema.id/transform] [:sequential ::lib.schema.id/transform]]]]
(when (seq transform-ids)
(into {}
(map (juxt :transform_id :last_start))
(transforms.db/last-start-times transform-ids))))
-(defn- status-labels
+(mu/defn- status-labels :- [:map-of :string :string]
"Display labels for TransformRun status values."
[]
{"started" (tru "In progress")
@@ -238,13 +280,13 @@
"canceling" (tru "Canceling")
"canceled" (tru "Canceled")})
-(defn- run-method-labels
+(mu/defn- run-method-labels :- [:map-of :string :string]
"Display labels for TransformRun run_method values."
[]
{"manual" (tru "Manual")
"cron" (tru "Schedule")})
-(defn- tag-name-labels
+(mu/defn- tag-name-labels :- [:map-of :string :string]
"Display labels for built-in TransformTag names."
[]
{"hourly" (tru "hourly")
@@ -252,12 +294,28 @@
"weekly" (tru "weekly")
"monthly" (tru "monthly")})
-(defn paged-runs
+(mu/defn paged-runs :- [:map {:closed true}
+ [:data [:sequential ::paged-run]]
+ [:limit ms/PositiveInt]
+ [:offset ms/IntGreaterThanOrEqualToZero]
+ [:total ms/IntGreaterThanOrEqualToZero]]
"Return a page of the list of the runs.
Follows the conventions used by the FE."
[{:keys [offset limit start-time end-time run-methods transform-ids transform-tag-ids statuses user-id
- sort-column sort-direction]}]
+ sort-column sort-direction]}
+ :- [:map {:closed true}
+ [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
+ [:limit {:optional true} [:maybe ms/PositiveInt]]
+ [:start-time {:optional true} [:maybe :string]]
+ [:end-time {:optional true} [:maybe :string]]
+ [:run-methods {:optional true} [:maybe [:sequential :string]]]
+ [:transform-ids {:optional true} [:maybe [:sequential ::lib.schema.id/transform]]]
+ [:transform-tag-ids {:optional true} [:maybe [:sequential ms/IntGreaterThanOrEqualToZero]]]
+ [:statuses {:optional true} [:maybe [:sequential :string]]]
+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:sort-column {:optional true} [:maybe [:or :keyword :string]]]
+ [:sort-direction {:optional true} [:maybe [:or :keyword :string]]]]]
(let [offset (or offset 0)
limit (or limit 20)
[started-at-start started-at-end] (when start-time (transforms.models.u/timestamp-range start-time))
diff --git a/src/metabase/transforms/models/transform_run_cancelation.clj b/src/metabase/transforms/models/transform_run_cancelation.clj
index aff0ab92b9d..c7e1339bbe8 100644
--- a/src/metabase/transforms/models/transform_run_cancelation.clj
+++ b/src/metabase/transforms/models/transform_run_cancelation.clj
@@ -3,6 +3,8 @@
[metabase.analytics-interface.core :as analytics]
[metabase.transforms.db :as transforms.db]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -12,9 +14,9 @@
(derive :model/TransformRunCancelation :metabase/model)
-(defn mark-cancel-started-run!
+(mu/defn mark-cancel-started-run! :- :nil
"Mark a started run for cancelation."
- [run-id]
+ [run-id :- ms/PositiveInt]
(try
(transforms.db/insert-cancelation-for-active-run! run-id)
(transforms.db/mark-run-canceling! run-id)
@@ -25,17 +27,17 @@
(analytics/inc! :metabase-transforms/cancelation-requests {:status "error"})
(throw t))))
-(defn reducible-canceled-local-runs
+(mu/defn reducible-canceled-local-runs
"Return a reducible sequence of local canceled runs."
[]
(transforms.db/cancelations-reducible))
-(defn delete-cancelation!
+(mu/defn delete-cancelation!
"Delete a cancelation once it has been handled."
- [run-id]
+ [run-id :- ms/PositiveInt]
(transforms.db/delete-cancelation-for-inactive-run! run-id))
-(defn delete-old-canceling-runs!
+(mu/defn delete-old-canceling-runs!
"Delete cancelations for runs that are no longer running."
[]
(transforms.db/delete-cancelations-for-inactive-runs!))
diff --git a/src/metabase/transforms/models/transform_schedule.clj b/src/metabase/transforms/models/transform_schedule.clj
index dc62558c893..4adfab7897d 100644
--- a/src/metabase/transforms/models/transform_schedule.clj
+++ b/src/metabase/transforms/models/transform_schedule.clj
@@ -1,22 +1,24 @@
(ns metabase.transforms.models.transform-schedule
"Cron schedules attached to transforms."
(:require
- [metabase.transforms.db :as transforms.db]))
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.transforms.db :as transforms.db]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
-(defn- schedules-by-transform-id
- [rows]
+(mu/defn- schedules-by-transform-id :- [:map-of ::lib.schema.id/transform [:set :string]]
+ [rows :- [:sequential [:map {:closed true} [:transform_id ::lib.schema.id/transform] [:schedule :string]]]]
(reduce (fn [m {:keys [transform_id schedule]}]
(update m transform_id (fnil conj #{}) schedule))
{}
rows))
-(defn schedules-for-transforms
+(mu/defn schedules-for-transforms :- [:maybe [:map-of ::lib.schema.id/transform [:set :string]]]
"Map each id in `transform-ids` — or every transform, in the 0-arity — to the cron schedules of the
active jobs that run it via shared tags. Ids with no such job are absent."
([]
(schedules-by-transform-id (transforms.db/active-job-schedules)))
- ([transform-ids]
+ ([transform-ids :- [:maybe [:or [:set ::lib.schema.id/transform] [:sequential ::lib.schema.id/transform]]]]
(when (seq transform-ids)
(schedules-by-transform-id (transforms.db/active-job-schedules-for-transforms transform-ids)))))
diff --git a/src/metabase/transforms/models/transform_tag.clj b/src/metabase/transforms/models/transform_tag.clj
index b69da3737f4..4c769b2ed58 100644
--- a/src/metabase/transforms/models/transform_tag.clj
+++ b/src/metabase/transforms/models/transform_tag.clj
@@ -6,8 +6,10 @@
[metabase.models.serialization :as serdes]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.models.transform :as transform]
- [metabase.transforms.schema]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.util.i18n :as i18n]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -38,17 +40,19 @@
[_model _instance]
(api/entitled-data-analyst?))
-(defn tag-name-exists?
+(mu/defn tag-name-exists? :- :boolean
"Check if a tag with the given name already exists"
- [tag-name]
+ [tag-name :- :string]
(transforms.db/tag-name-exists? tag-name))
-(defn tag-name-exists-excluding?
+(mu/defn tag-name-exists-excluding? :- :boolean
"Check if a tag with the given name exists, excluding the specified ID"
- [tag-name tag-id]
+ [tag-name :- :string
+ tag-id :- ms/PositiveInt]
(transforms.db/tag-name-exists-excluding? tag-name tag-id))
-(defn- translate-name [tag]
+(mu/defn- translate-name :- [:map {:closed true} [:name [:maybe mu/localized-string-schema]]]
+ [tag :- [:or ::transforms.schema/transform-tag ::transforms.schema/transform-tag.update]]
(let [values {"hourly" (i18n/deferred-trs "hourly")
"daily" (i18n/deferred-trs "daily")
"weekly" (i18n/deferred-trs "weekly")
diff --git a/src/metabase/transforms/models/util.clj b/src/metabase/transforms/models/util.clj
index 876fb3ecb50..ed6e600fb16 100644
--- a/src/metabase/transforms/models/util.clj
+++ b/src/metabase/transforms/models/util.clj
@@ -2,14 +2,16 @@
(:require
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.util.date-2 :as u.date]
- [metabase.util.i18n :refer [tru]]))
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
-(defn timestamp-range
+(mu/defn timestamp-range :- [:tuple [:maybe ms/TemporalInstant] [:maybe ms/TemporalInstant]]
"Parse `date-string` (in the same format as the query-processor date parameter, e.g. \"2025-01\",
\"past7days\") into a `[start end]` pair of instants (either may be nil)."
- [date-string]
+ [date-string :- :string]
(let [{:keys [start end]}
(try
(params.dates/date-string->range date-string {:inclusive-end? false})
@@ -19,10 +21,11 @@
e))))]
[(some-> start u.date/parse) (some-> end u.date/parse)]))
-(defn timestamp-constraint
+(mu/defn timestamp-constraint :- [:cat [:= :and] [:* [:or [:tuple [:= :>=] :keyword ms/TemporalInstant] [:tuple [:= :<] :keyword ms/TemporalInstant]]]]
"Build a HoneySQL `[:and ...]` clause constraining `field-name` to the range expressed by `date-string`.
`date-string` uses the same format as the query-processor date parameter (e.g. \"2025-01\", \"past7days\")."
- [field-name date-string]
+ [field-name :- :keyword
+ date-string :- :string]
(let [[start end] (timestamp-range date-string)]
(into [:and] (remove nil?)
[(when start [:>= field-name start])
diff --git a/src/metabase/transforms/notification.clj b/src/metabase/transforms/notification.clj
index 2d417590ffb..49493439773 100644
--- a/src/metabase/transforms/notification.clj
+++ b/src/metabase/transforms/notification.clj
@@ -12,18 +12,34 @@
[metabase.task.core :as task]
[metabase.transforms.db :as transforms.db]
[metabase.util.date-2 :as u.date]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
-(defn- recent-failed-cron-job-runs
+(mr/def ::failing-job
+ [:map {:closed true}
+ [:job_name [:maybe :string]]
+ [:job_href :string]
+ [:failure_count :int]
+ [:first_failed [:maybe :string]]
+ [:latest_error [:maybe :string]]])
+
+(mu/defn- recent-failed-cron-job-runs :- [:sequential [:map {:closed true}
+ [:job_id ms/PositiveInt]
+ [:start_time [:maybe ms/TemporalInstant]]
+ [:message [:maybe :string]]]]
"Cron job runs that failed or timed out in `[start, end)`, oldest first."
- [start end]
+ [start :- ms/TemporalInstant
+ end :- ms/TemporalInstant]
(transforms.db/failed-cron-job-runs-between start end))
-(defn failing-jobs
+(mu/defn failing-jobs :- [:sequential ::failing-job]
"Summarize failed/timed-out cron job runs in `[start, end)`, one entry per job."
- [start end]
+ [start :- ms/TemporalInstant
+ end :- ms/TemporalInstant]
(let [runs (recent-failed-cron-job-runs start end)
job-ids (distinct (map :job_id runs))
id->name (when (seq job-ids)
@@ -39,14 +55,17 @@
(sort-by :first_failed)
vec)))
-(defn- digest-timezone
+(mu/defn- digest-timezone :- (ms/InstanceOfClass java.time.ZoneId)
"Zone for the digest's calendar-day boundaries, matching transform scheduling."
[]
(t/zone-id (or (driver/report-timezone)
(qp.timezone/system-timezone-id)
"UTC")))
-(defn digest-info
+(mu/defn digest-info :- [:map {:closed true}
+ [:job_count :int]
+ [:failure_count :int]
+ [:jobs [:sequential ::failing-job]]]
"Event info for the failure digest, covering the previous calendar day."
[]
(let [yesterday (u.date/add (t/zoned-date-time (digest-timezone)) :day -1)
@@ -56,13 +75,14 @@
:failure_count (reduce + 0 (map :failure_count jobs))
:jobs jobs}))
-(defn send-failure-digest!
+(mu/defn send-failure-digest! :- :nil
"Publish the daily digest event, unless there were no failures."
[]
(let [{:keys [jobs] :as info} (digest-info)]
(if (seq jobs)
(events/publish-event! :event/transform-failure-digest info)
- (log/info "No transform job failures yesterday; skipping digest."))))
+ (log/info "No transform job failures yesterday; skipping digest."))
+ nil))
(def ^:private digest-job-key "metabase.transforms.notification.failure-digest-job")
(def ^:private digest-trigger-key "metabase.transforms.notification.failure-digest-trigger")
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index 0943d7f802e..0ed1f0d9d43 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -7,18 +7,22 @@
[metabase.request.core :as request]
[metabase.tracing.core :as tracing]
[metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.interface :as transforms.i]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
-(defn- run-mbql-transform!
- ([transform] (run-mbql-transform! transform nil))
- ([{:keys [id source target owner_user_id creator_id] :as transform}
- {:keys [run-method on-start user-id parent-run]}]
+(mu/defn- run-mbql-transform! :- :nil
+ ([transform :- ::transforms-base.schema/transform]
+ (run-mbql-transform! transform nil))
+ ([{:keys [id source target owner_user_id creator_id] :as transform} :- ::transforms-base.schema/transform
+ {:keys [run-method on-start user-id parent-run]} :- [:maybe ::transforms.schema/execute-options]]
(try
(let [db (driver.u/lib-database (get-in source [:query :database]))
driver (:engine db)
@@ -67,7 +71,8 @@
(throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
result))))]
;; Post-processing: sync, transform_id, events
- (transforms-base.u/complete-execution! transform {}))))))
+ (transforms-base.u/complete-execution! transform {})))))
+ nil)
(catch Throwable t
(if (= :already-running (:error (ex-data t)))
(log/warnf "Transform %d is already running" id)
diff --git a/src/metabase/transforms/run_listing.clj b/src/metabase/transforms/run_listing.clj
index d9a0600b531..6c6ade96ea8 100644
--- a/src/metabase/transforms/run_listing.clj
+++ b/src/metabase/transforms/run_listing.clj
@@ -9,20 +9,45 @@
their parent run. Because ids are per-table, a row is identified by the `(run_type, id)` pair;
`entity_id` is the id of the associated job/transform."
(:require
- [medley.core :as m]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.db :as transforms.db]
- [metabase.transforms.models.util :as transforms.models.u]))
+ [metabase.transforms.models.util :as transforms.models.u]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
(set! *warn-on-reflection* true)
+(mr/def ::run-summary.response
+ "A [[::transforms.schema/run-summary]] as [[present-run-summaries]] prepares it for an API response."
+ [:map {:closed true}
+ [:run_type [:enum :job :dag :transform]]
+ [:id ms/PositiveInt]
+ [:entity_id [:maybe ms/PositiveInt]]
+ [:name [:maybe :string]]
+ [:direction [:maybe [:enum :upstream :downstream]]]
+ [:transform_count [:maybe :int]]
+ [:run_method [:maybe :keyword]]
+ [:status :keyword]
+ [:is_active [:maybe :boolean]]
+ [:start_time [:maybe :string]]
+ [:end_time {:optional true} [:maybe :string]]
+ [:message [:maybe :string]]
+ [:user_id [:maybe ::lib.schema.id/user]]])
+
(def run-types
"The `run_type` discriminator values a listing row can have."
[:job :dag :transform])
-(defn paged-run-summaries
- "Return a page of root runs as a `{:data :limit :offset :total}` envelope. Rows are raw — see
- [[present-run-summaries]].
+(mu/defn paged-run-summaries :- [:map {:closed true}
+ [:data [:sequential ::transforms.schema/run-summary]]
+ [:limit ms/PositiveInt]
+ [:offset ms/IntGreaterThanOrEqualToZero]
+ [:total :int]]
+ "Return a page of root runs as a `{:data :limit :offset :total}` envelope; [[present-run-summaries]] prepares its
+ rows for an API response.
Options (filter semantics match [[metabase.transforms.models.transform-run/paged-runs]]):
- `:types` subset of [[run-types]] to include; nil/empty means all three
@@ -32,10 +57,21 @@
- `:end-time` likewise for `end_time`
- `:transform-ids` only runs that ran any of these transforms (job/DAG runs whose members include
one, or the standalone runs of them)
- - `:sort-column` `\"start_time\"` / `\"end_time\"`
- - `:sort-direction` `\"asc\"` / `\"desc\"`
+ - `:sort-column` `:start_time` / `:end_time`
+ - `:sort-direction` `:asc` / `:desc`
- `:offset`/`:limit` pagination (default 0 / 20)"
- [{:keys [types statuses run-methods start-time end-time transform-ids sort-column sort-direction offset limit]}]
+ [{:keys [types statuses run-methods start-time end-time transform-ids sort-column sort-direction offset limit]}
+ :- [:map {:closed true}
+ [:types {:optional true} [:maybe [:sequential [:enum :job :dag :transform]]]]
+ [:statuses {:optional true} [:maybe [:sequential :string]]]
+ [:run-methods {:optional true} [:maybe [:sequential :string]]]
+ [:start-time {:optional true} [:maybe ms/NonBlankString]]
+ [:end-time {:optional true} [:maybe ms/NonBlankString]]
+ [:transform-ids {:optional true} [:maybe [:sequential ::lib.schema.id/transform]]]
+ [:sort-column {:optional true} [:maybe [:enum :start_time :end_time]]]
+ [:sort-direction {:optional true} [:maybe [:enum :asc :desc]]]
+ [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
+ [:limit {:optional true} [:maybe ms/PositiveInt]]]]
(let [offset (or offset 0)
limit (or limit 20)
[start-at end-at] (when start-time (transforms.models.u/timestamp-range start-time))
@@ -47,25 +83,21 @@
:total (transforms.db/root-run-summaries-count types statuses run-methods start-at end-at ended-at end-end
transform-ids)}))
-(defn present-run-summaries
- "Prepare raw summary rows for an API response: hydrate each row's `:name` (nil when the underlying
- job/transform was deleted), keywordize the discriminator columns, and localize timestamps."
- [rows]
+(mu/defn present-run-summaries :- [:sequential ::run-summary.response]
+ "Prepare summary rows for an API response: hydrate each row's `:name` (nil when the underlying job/transform was
+ deleted) and localize timestamps."
+ [rows :- [:sequential ::transforms.schema/run-summary]]
(let [by-type (group-by :run_type rows)
- job-ids (seq (keep :entity_id (get by-type "job")))
- transform-ids (seq (keep :entity_id (concat (get by-type "dag") (get by-type "transform"))))
+ job-ids (seq (keep :entity_id (get by-type :job)))
+ transform-ids (seq (keep :entity_id (concat (get by-type :dag) (get by-type :transform))))
job->name (when job-ids (transforms.db/job-names-by-id job-ids))
transform->name (when transform-ids (transforms.db/transform-names-by-id transform-ids))]
(map (fn [{:keys [run_type entity_id entity_name] :as row}]
(-> row
- (assoc :name (or (if (= run_type "job")
+ (assoc :name (or (if (= run_type :job)
(get job->name entity_id)
(get transform->name entity_id))
entity_name))
(dissoc :entity_name)
- (update :run_type keyword)
- (update :status keyword)
- (m/update-existing :run_method #(some-> % keyword))
- (m/update-existing :direction #(some-> % keyword))
transforms-base.u/localize-run-timestamps))
rows)))
diff --git a/src/metabase/transforms/schedule.clj b/src/metabase/transforms/schedule.clj
index 1ae6f74bcd7..a071248018f 100644
--- a/src/metabase/transforms/schedule.clj
+++ b/src/metabase/transforms/schedule.clj
@@ -10,36 +10,62 @@
[metabase.task.core :as task]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.jobs :as transforms.jobs]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.settings :as transforms.settings]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms])
(:import
(java.util TimeZone)
- (org.quartz CronTrigger TriggerKey CronExpression)))
+ (org.quartz CronTrigger CronExpression JobKey TriggerKey)))
(set! *warn-on-reflection* true)
-(defn validate-cron-expression
+(mr/def ::trigger-info
+ "A Quartz trigger as `metabase.task.core/existing-triggers` describes it."
+ [:map {:closed true}
+ [:description [:maybe :string]]
+ [:end-time [:maybe inst?]]
+ [:final-fire-time [:maybe inst?]]
+ [:key :string]
+ [:state [:maybe :string]]
+ [:next-fire-time [:maybe inst?]]
+ [:previous-fire-time [:maybe inst?]]
+ [:priority :int]
+ [:start-time [:maybe inst?]]
+ [:may-fire-again? :boolean]
+ [:data [:map-of :string :string]]
+ [:schedule :string]
+ [:timezone :string]
+ [:misfire-instruction :string]])
+
+(mu/defn validate-cron-expression :- :boolean
"Returns true if valid, false otherwise."
- [expr]
+ [expr :- :string]
(try
;; throwing is its only mechanism:
(CronExpression/validateExpression expr)
true
(catch Exception _ false)))
-(defn- job-key [job-id] (jobs/key (str "metabase.task.transforms.schedule." job-id)))
+(mu/defn- job-key :- (ms/InstanceOfClass JobKey)
+ [job-id :- ms/PositiveInt]
+ (jobs/key (str "metabase.task.transforms.schedule." job-id)))
-(defn- timezone
+(mu/defn- timezone :- :string
[]
(or (driver/report-timezone)
(qp.timezone/system-timezone-id)
"UTC"))
-(defn- trigger-key ^TriggerKey [job-id]
+(mu/defn- trigger-key :- (ms/InstanceOfClass TriggerKey)
+ [job-id :- ms/PositiveInt]
(triggers/key (str "metabase.task.transforms.trigger." job-id)))
-(defn- build-trigger
- ^CronTrigger [job-id schedule]
+(mu/defn- build-trigger :- (ms/InstanceOfClass CronTrigger)
+ [job-id :- ms/PositiveInt
+ schedule :- :string]
(triggers/build
(triggers/with-description (str "Transform Job Schedule " job-id))
(triggers/with-identity (trigger-key job-id))
@@ -55,23 +81,24 @@
;; higher than sync
(triggers/with-priority 6)))
-(defn- create-trigger!
+(mu/defn- create-trigger! :- [:maybe inst?]
"Creates the trigger for a transform job."
- [job-id schedule]
+ [job-id :- ms/PositiveInt
+ schedule :- [:maybe :string]]
(when schedule
(log/info "Creating trigger for transform job" job-id "with schedule" schedule)
(task/add-trigger! (build-trigger job-id schedule))))
-(defn existing-trigger
+(mu/defn existing-trigger :- [:maybe ::trigger-info]
"Return the existing trigger for `job-id-or-trigger`, or nil, if the trigger doesn't exist."
- [job-id-or-trigger]
+ [job-id-or-trigger :- ms/PositiveInt]
(first (task/existing-triggers (job-key job-id-or-trigger)
(trigger-key job-id-or-trigger))))
-(defn delete-trigger!
+(mu/defn delete-trigger! :- [:maybe :boolean]
"Delete the trigger for a transform job. Leaves the underlying Quartz job definition intact so it
can be re-scheduled later via [[initialize-job!]]."
- ([job-id-or-trigger]
+ ([job-id-or-trigger :- [:or ms/PositiveInt ::trigger-info]]
(if (number? job-id-or-trigger)
(if-let [trigger (existing-trigger job-id-or-trigger)]
(delete-trigger! trigger)
@@ -104,9 +131,9 @@
;; which is misleading. The task_history row above still records the skipped firing.
(log/info "Skipping scheduled run of transform job" job-id "because transforms are disabled")))))
-(defn initialize-job!
+(mu/defn initialize-job! :- [:maybe inst?]
"Initialize a schedule for a transform job."
- [{job-id :id :keys [schedule]}]
+ [{job-id :id :keys [schedule]} :- ::transforms.schema/transform-job]
(log/info "Initializing schedule for transform job" job-id)
(let [job (jobs/build
(jobs/with-identity (job-key job-id))
@@ -117,9 +144,10 @@
trigger (build-trigger job-id schedule)]
(task/schedule-task! job trigger)))
-(defn update-job!
+(mu/defn update-job! :- [:maybe inst?]
"Update the trigger for a transform job."
- [job-id schedule]
+ [job-id :- ms/PositiveInt
+ schedule :- [:maybe :string]]
(log/info "Updating schedule for transform job" job-id "to" schedule)
(let [existing-trigger (first (task/existing-triggers (job-key job-id) (trigger-key job-id)))]
(if (not= schedule (:schedule existing-trigger))
@@ -130,9 +158,9 @@
(create-trigger! job-id schedule)))
(log/info "No changes to the trigger for transform job" job-id))))
-(defn delete-job!
+(mu/defn delete-job! :- [:maybe :boolean]
"Delete a job."
- [job-id]
+ [job-id :- ms/PositiveInt]
(log/info "Deleting schedule for transform job" job-id)
(task/delete-task! (job-key job-id) (trigger-key job-id)))
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index da22d706c1c..e13996b99ba 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -1,5 +1,6 @@
(ns metabase.transforms.schema
(:require
+ [metabase.indexes.schema :as indexes.schema]
[metabase.lib-be.schema :as lib-be.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
@@ -13,7 +14,7 @@
"A lookback window: each run re-reads source rows up to `value` `unit`s behind the checkpoint,
so late-arriving rows older than the watermark still get picked up. Only supported for
temporal checkpoint columns."
- [:map {:closed true}
+ [:map {:closed true, :decode/normalize lib.schema.common/normalize-map-no-kebab-case}
[:value pos-int?]
[:unit (into [:enum] (map name) (sort u.date/add-units))]])
@@ -28,32 +29,31 @@
:dispatch :type}
["checkpoint" ::checkpoint-strategy]])
-(mr/def ::orphaned-query
- "The MBQL 5 query of a transform whose source database was deleted, kept as a breadcrumb with its `:database` nulled."
+(mr/def ::query-source
+ "The `:source` of a query transform: its MBQL 5 query, normalized from legacy MBQL where a client or a serdes export
+ still writes it."
+ [:map {:closed true}
+ [:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :query]]
+ [:query ::lib-be.schema/maybe-legacy-query]
+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]])
+
+(mr/def ::python-source
+ "The `:source` of a Python transform: its script and the tables it reads."
[:map {:closed true}
- [:lib/type [:= {:decode/normalize lib.schema.common/normalize-keyword} :mbql/query]]
- [:database :nil]
- [:stages [:ref :metabase.lib.schema/stages]]
- [:lib/metadata {:optional true} [:ref :metabase.lib.schema.metadata/metadata-provider]]])
+ [:source-database {:optional true} [:maybe :int]]
+ ;; NB: if source is checkpoint, only one table allowed
+ [:source-tables {:decode/normalize #(cond-> % (map? %) (-> (update-keys name) transforms-base.u/source-tables-map->vec))}
+ [:sequential ::transforms-base.u/source-table-entry]]
+ [:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]]
+ [:body :string]
+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]])
(mr/def ::transform-source
+ "The `:source` of a transform, by its `:type`."
[:multi {:decode/normalize lib.schema.common/normalize-map-no-kebab-case
:dispatch (comp keyword :type)}
- [:query
- [:map {:closed true}
- [:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :query]]
- [:query [:or
- ::orphaned-query
- ::lib-be.schema/maybe-legacy-query]]
- [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]
- [:python
- [:map {:closed true}
- [:source-database {:optional true} :int]
- ;; NB: if source is checkpoint, only one table allowed
- [:source-tables [:sequential ::transforms-base.u/source-table-entry]]
- [:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]]
- [:body :string]
- [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])
+ [:query ::query-source]
+ [:python ::python-source]])
(mr/def ::append-config
[:map {:closed true} [:type [:= "append"]]])
@@ -109,9 +109,7 @@
(mr/def ::transform.table-dependency
"One entry of the `:table_dependencies` column of a Transform, decoded."
- [:or
- [:map {:closed true} [:table ::lib.schema.id/table]]
- [:map {:closed true} [:transform ::lib.schema.id/transform]]])
+ :metabase.transforms-base.schema/table-dependency)
(mr/def ::transform.owner
"The `:owner` hydrated onto a Transform: its owning User, or just the `:email` of an owner outside Metabase."
@@ -119,9 +117,40 @@
:metabase.users.schema/user
[:map {:closed true} [:email :string]]])
+(mr/def ::localized-label
+ "A user-facing label: a string, or one translated when it is rendered."
+ [:or :string mu/localized-string-schema])
+
+(mr/def ::requestable-index-field
+ "One form field for requesting an index of some kind, as a driver describes it."
+ [:map {:closed true}
+ [:name :string]
+ [:display-name ::localized-label]
+ [:description {:optional true} ::localized-label]
+ [:type [:enum :string :boolean :select :integer :columns]]
+ [:required {:optional true} :boolean]
+ [:directions {:optional true} :boolean]
+ [:options {:optional true} [:sequential [:map {:closed true}
+ [:name ::localized-label]
+ [:value :string]]]]])
+
+(mr/def ::requestable-index-method
+ "One kind of index a Transform's target table can have, as its driver describes it."
+ [:map {:closed true}
+ [:display-name ::localized-label]
+ [:description {:optional true} ::localized-label]
+ [:lifecycle [:enum :standalone :inline]]
+ [:fields [:sequential ::requestable-index-field]]])
+
+(mr/def ::requestable-indexes
+ "The kinds of index a Transform's target table can have, by kind."
+ (into [:map {:closed true}]
+ (for [kind indexes.schema/index-kinds]
+ [kind {:optional true} ::requestable-index-method])))
+
(mr/def ::transform
- "A Transform as selected from the app DB: every column of `:transform`, plus `:creator`, `:table`, `:last_run`,
- `:collection`, and `:owner` some callers hydrate onto it."
+ "A Transform as selected from the app DB: every column of `:transform`, plus what callers hydrate onto it or add for
+ an API response."
[:merge
::transform.update
[:map {:closed true}
@@ -134,7 +163,13 @@
[:can_read {:optional true} :boolean]
[:can_write {:optional true} :boolean]
[:can_execute {:optional true} :boolean]
- [:tag_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]]])
+ [:tag_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]
+ [:source_readable {:optional true} :boolean]
+ [:requestable_indexes {:optional true} [:maybe ::requestable-indexes]]]])
+
+(mr/def ::run-trigger
+ "What starts a Transform run: nothing, or the global schedule."
+ (ms/enum-decode-keyword [:none :global-schedule]))
(mr/def ::transform.update
"What an update (or insert) of a Transform accepts: every column of `:transform` except `id`, all optional, plus `:run_trigger` consumed by the model's hooks."
@@ -156,7 +191,15 @@
[:last_checkpoint_value {:optional true} [:maybe :string]]
[:target_table_id {:optional true} [:maybe ::lib.schema.id/table]]
[:table_dependencies {:optional true} [:maybe [:sequential ::transform.table-dependency]]]
- [:run_trigger {:optional true} [:maybe [:or :keyword :string]]]])
+ [:run_trigger {:optional true} [:maybe ::run-trigger]]])
+
+(mr/def ::transform.definition
+ "What a Transform computes and where it writes, as a client writes it."
+ [:map {:closed true}
+ [:name :string]
+ [:description {:optional true} [:maybe :string]]
+ [:source ::transform-source]
+ [:target ::transform-target]])
(mr/def ::transform-dag-run
"A TransformDagRun as selected from the app DB: every column of `:transform_dag_run`."
@@ -311,3 +354,58 @@
[:tag_id {:optional true} [:maybe ms/PositiveInt]]
[:entity_id {:optional true} [:maybe :string]]
[:position {:optional true} [:maybe :int]]])
+
+(mr/def ::coordinated-run-model
+ "A model that tracks a coordinated run."
+ [:enum :model/TransformJobRun :model/TransformDagRun])
+
+(mr/def ::transform-plan
+ "An execution plan: the Transforms in execution order, and the dependencies they were ordered by."
+ [:map {:closed true}
+ [:order [:sequential ::transform]]
+ [:deps [:map-of ::lib.schema.id/transform [:set ::lib.schema.id/transform]]]])
+
+(mr/def ::promise
+ "A promise."
+ (ms/InstanceOfClass clojure.lang.IPending))
+
+(mr/def ::transform.create
+ "What a client posts to create a Transform: the columns an insert accepts, plus the `:tag_ids` to link."
+ [:merge
+ ::transform.update
+ [:map {:closed true}
+ [:tag_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]]])
+
+(mr/def ::transform-run.presented
+ "A TransformRun prepared for an API response: its timestamps are ISO strings in UTC."
+ [:merge
+ ::transform-run
+ [:map {:closed true}
+ [:start_time {:optional true} [:maybe :string]]
+ [:end_time {:optional true} [:maybe :string]]]])
+
+(mr/def ::execute-options
+ "The options of `transforms.i/execute!`."
+ [:map {:closed true}
+ [:run-method {:optional true} [:maybe [:enum :manual :cron]]]
+ [:on-start {:optional true} [:maybe [:=> [:cat ms/PositiveInt] :nil]]]
+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:parent-run {:optional true} [:maybe [:tuple [:enum :job :dag] ms/PositiveInt]]]
+ [:start-promise {:optional true} [:maybe ::promise]]])
+
+(mr/def ::run-summary
+ "A root run of the paged run listing: a job run, a manual DAG run, or a standalone transform run."
+ [:map {:closed true}
+ [:run_type (ms/enum-decode-keyword [:job :dag :transform])]
+ [:id ms/PositiveInt]
+ [:entity_id [:maybe ms/PositiveInt]]
+ [:entity_name [:maybe :string]]
+ [:direction [:maybe (ms/enum-decode-keyword [:upstream :downstream])]]
+ [:transform_count [:maybe :int]]
+ [:run_method [:maybe (ms/enum-decode-keyword [:manual :cron])]]
+ [:status (ms/enum-decode-keyword [:started :succeeded :failed :timeout :canceled :canceling])]
+ [:is_active [:maybe :boolean]]
+ [:start_time [:maybe ms/TemporalInstant]]
+ [:end_time {:optional true} [:maybe ms/TemporalInstant]]
+ [:message [:maybe :string]]
+ [:user_id [:maybe ::lib.schema.id/user]]])
diff --git a/src/metabase/transforms/settings.clj b/src/metabase/transforms/settings.clj
index 888c9c901a0..204ef570b0c 100644
--- a/src/metabase/transforms/settings.clj
+++ b/src/metabase/transforms/settings.clj
@@ -3,7 +3,8 @@
[metabase.premium-features.core :as premium-features]
[metabase.settings.core :as setting]
[metabase.transforms.usage :as transforms.usage]
- [metabase.util.i18n :refer [deferred-tru]]))
+ [metabase.util.i18n :refer [deferred-tru]]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -37,12 +38,12 @@
:encryption :no
:audit :getter)
-(defn- explicit-transforms-enabled
+(mu/defn- explicit-transforms-enabled :- [:maybe :boolean]
"Returns the :transforms-enabled setting if explicitly set, otherwise nil."
[]
(setting/get-value-of-type :boolean :transforms-enabled))
-(defn- cloud-enabled-transforms?
+(mu/defn- cloud-enabled-transforms? :- :boolean
"Whether a cloud instance has enabled transforms.
If a cloud instance has the transforms token feature, then they've gone through the upsell and enabled transforms.
Self-hosted instances don't go through the upsell, so we require that they explicitly enable transforms via the setting."
diff --git a/src/metabase/transforms/timeout.clj b/src/metabase/transforms/timeout.clj
index b411d907820..b5f0a02b9f0 100644
--- a/src/metabase/transforms/timeout.clj
+++ b/src/metabase/transforms/timeout.clj
@@ -7,16 +7,22 @@
[metabase.tracing.core :as tracing]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.settings :as transforms.settings]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms])
+ (:import
+ (org.quartz JobExecutionContext)))
(set! *warn-on-reflection* true)
(def ^:private job-key "metabase.transforms.timeout")
-(defn- timeout-transform-runs! [_ctx]
+(mu/defn- timeout-transform-runs! :- :nil
+ [_ctx :- (ms/InstanceOfClass JobExecutionContext)]
(tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "transform"}
(when-let [timed-out (not-empty (transform-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute))]
- (log/infof "Timed out %d transform run(s)." (count timed-out)))))
+ (log/infof "Timed out %d transform run(s)." (count timed-out))))
+ nil)
(task/defjob TimeoutTransforms
"Timeout long-running tasks that have been lost by a worker."
@@ -25,7 +31,8 @@
[ctx]
(timeout-transform-runs! ctx))
-(defn- start-job! []
+(mu/defn- start-job! :- :nil
+ []
(when (not (task/job-exists? job-key))
(let [job (jobs/build
(jobs/of-type TimeoutTransforms)
@@ -37,7 +44,8 @@
(calendar-interval/schedule
(calendar-interval/with-interval-in-minutes 10)
(calendar-interval/with-misfire-handling-instruction-do-nothing))))]
- (task/schedule-task! job trigger))))
+ (task/schedule-task! job trigger)))
+ nil)
(defmethod task/init! ::TimeoutTransforms [_]
(log/info "Scheduling timeout transforms task.")
diff --git a/src/metabase/transforms/usage.clj b/src/metabase/transforms/usage.clj
index 73c446a32e0..403b7fb1ad0 100644
--- a/src/metabase/transforms/usage.clj
+++ b/src/metabase/transforms/usage.clj
@@ -2,7 +2,9 @@
"Meter-related limit checking for transforms"
(:require
[metabase.premium-features.core :as premium-features]
- [metabase.transforms-base.util :as transforms-base.u]))
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]))
(def ^:private metered-as->meter-key
"Map the bucket strings returned by [[premium-features/transform-metered-as]] to their
@@ -10,11 +12,11 @@
{"transform-basic" :transform-basic-runs
"transform-advanced" :transform-advanced-runs})
-(defn transform-locked?
+(mu/defn transform-locked? :- :boolean
"True if the meter that would be charged for running this transform is locked.
Returns false for non-metered transforms (OSS, self-hosted basic-only) and when
no `:locked-meters` mirror has been written yet (cold cache, airgap)."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(boolean
(when-let [meter-key (some-> (:source transform)
transforms-base.u/transform-source-type
@@ -22,7 +24,7 @@
metered-as->meter-key)]
(get (premium-features/locked-meters) meter-key))))
-(defn transforms-meter-locked?
+(mu/defn transforms-meter-locked? :- :boolean
"True if either of the two transforms meters (`:transform-basic-runs` or
`:transform-advanced-runs`) is currently locked. Per harbormaster's
mutual-exclusivity constraint, at most one of these is populated for a
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index e6436e95be3..3b76ebbcbe5 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -13,6 +13,7 @@
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.util :as driver.u]
[metabase.indexes.models.table-index :as table-index]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.premium-features.core :as premium-features]
[metabase.query-processor.middleware.permissions :as qp.perms]
@@ -20,34 +21,50 @@
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.setup :as qp.setup]
[metabase.tracing.core :as tracing]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as canceling]
[metabase.transforms.db :as transforms.db]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.models.transform-run :as transform-run]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.settings :as transforms.settings]
[metabase.util :as u]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms])
(:import
(java.sql SQLException)))
(set! *warn-on-reflection* true)
+(mr/def ::source-models
+ "The source Databases and Tables of some transforms by ID, as [[prefetch-source-models]] loads them."
+ [:map {:closed true}
+ [:model/Database [:maybe [:map-of ::lib.schema.id/database :metabase.warehouses.schema/database]]]
+ [:model/Table [:maybe [:map-of ::lib.schema.id/table :metabase.warehouse-schema.schema/table]]]])
+
+(mu/defn ->definition :- ::transforms.schema/transform.definition
+ "What the saved `transform`, or the one a create request carries, computes and where it writes."
+ [transform :- [:or ::transforms.schema/transform ::transforms.schema/transform.create]]
+ (select-keys transform [:name :description :source :target]))
+
;;; ------------------------------------------------- Feature Gating -------------------------------------------------
-(defn check-feature-enabled
+(mu/defn check-feature-enabled :- :boolean
"Checking whether we have proper feature flags for using a given transform."
- [transform]
+ [transform :- ::transforms.schema/transform.definition]
(cond
(transforms-base.u/query-transform? transform) (premium-features/query-transforms-enabled?)
(transforms-base.u/python-transform? transform) (premium-features/python-transforms-enabled?)
:else false))
-(defn- source-query-permissions-ok?
+(mu/defn- source-query-permissions-ok? :- :boolean
"Whether the current user may run a query transform's source `query`, per the query processor's own permission
check. The query is preprocessed first so references that only appear after expansion (cards, snippets) are
checked too. Throws when no user is bound: every caller must establish one, a scheduled run included."
- [query]
+ [query :- :metabase.lib.schema/query]
(when-not api/*current-user-id*
(throw (ex-info "A transform needs to be run with a bound user"
{})))
@@ -63,13 +80,15 @@
(not (or (:permissions-error? data)
(= 403 (:status-code data))))))))
-(defn source-tables-readable?
+(mu/defn source-tables-readable? :- :boolean
"Check if the source tables/database in a transform are readable by the current user.
Returns true if the user can query all source tables (for python transforms) or the
source database (for query transforms). Returns false if the referenced source database
no longer exists."
- ([transform] (source-tables-readable? transform nil))
- ([transform models-cache]
+ ([transform :- ::transforms.schema/transform.definition]
+ (source-tables-readable? transform nil))
+ ([transform :- ::transforms.schema/transform.definition
+ models-cache :- [:maybe ::source-models]]
(let [resolve* (fn [model id]
(if models-cache
(get-in models-cache [model id])
@@ -91,29 +110,30 @@
(if (empty? source-tables)
true
(let [table-ids (into [] (keep :table_id) source-tables)]
- (and (seq table-ids)
- (every? (fn [table-id]
- (when-let [table (resolve* :model/Table table-id)]
- (mi/can-query? table)))
- table-ids)))))
+ (boolean
+ (and (seq table-ids)
+ (every? (fn [table-id]
+ (when-let [table (resolve* :model/Table table-id)]
+ (mi/can-query? table)))
+ table-ids))))))
(throw (ex-info (str "Unknown transform source type: " (:type source)) {}))))))
-(defn check-source-query-permissions!
+(mu/defn check-source-query-permissions! :- :nil
"Throw a 403 unless the current user may run `transform`'s source query -- see
[[source-query-permissions-ok?]]. Requires a bound user; the execution path binds the user the run
executes as before calling.
Transforms compile and run their source query directly rather than through `qp.execute/run`, so the check is
made here."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(api/check-403 (source-query-permissions-ok? (get-in transform [:source :query])))
nil)
-(defn prefetch-source-models
+(mu/defn prefetch-source-models :- ::source-models
"Bulk-load the source databases and tables referenced by `transforms` into a
`{:model/Database {id db} :model/Table {id table}}` map"
- [transforms]
+ [transforms :- [:sequential ::transforms.schema/transform]]
(let [db-ids (into #{} (keep #(get-in % [:source :query :database])) transforms)
table-ids (into #{} (mapcat #(keep :table_id (get-in % [:source :source-tables]))) transforms)]
{:model/Database (when (seq db-ids)
@@ -121,34 +141,38 @@
:model/Table (when (seq table-ids)
(u/index-by :id (transforms.db/tables table-ids)))}))
-(defn add-source-readable
+(mu/defn add-source-readable :- [:or ::transforms.schema/transform [:sequential ::transforms.schema/transform]]
"Add :source_readable field to a transform or collection of transforms.
The field indicates whether the current user can read the source tables/database
referenced by the transform."
- [transform-or-transforms]
+ [transform-or-transforms :- [:or ::transforms.schema/transform [:sequential ::transforms.schema/transform]]]
(if (sequential? transform-or-transforms)
- (mapv #(assoc % :source_readable (source-tables-readable? %))
+ (mapv #(assoc % :source_readable (source-tables-readable? (->definition %)))
transform-or-transforms)
- (assoc transform-or-transforms :source_readable (source-tables-readable? transform-or-transforms))))
+ (assoc transform-or-transforms :source_readable (source-tables-readable? (->definition transform-or-transforms)))))
;;; ------------------------------------------------- Scheduled Execution -------------------------------------------------
-(defn duplicate-key-violation?
+(mu/defn duplicate-key-violation? :- :boolean
"Check if an exception is a duplicate key violation.
Returns true for Postgres, MySQL/MariaDB, and H2 duplicate key errors."
- [e]
+ [e :- [:maybe (ms/InstanceOfClass Throwable)]]
(or (and (instance? SQLException e)
(let [sql-state (sql-jdbc/get-sql-state e)]
(str/starts-with? sql-state "23")))
(some-> (ex-cause e) duplicate-key-violation?)))
-(defn try-start-unless-already-running
+(mu/defn try-start-unless-already-running :- ::transforms.schema/transform-run
"Start a transform run. Throws ex-info with {:error :already-running} if another
run is already active (duplicate key violation). Other errors are rethrown.
If `user-id` is provided, it will be stored with the run for attribution purposes.
`parent-run`, when provided, links the run to its coordinating run as a `[type id]` tuple:
`[:job id]` is stored in `job_run_id`, `[:dag id]` in `dag_run_id`."
- [id run-method user-id & {:keys [parent-run]}]
+ [id :- ::lib.schema.id/transform
+ run-method :- [:enum :manual :cron]
+ user-id :- [:maybe ::lib.schema.id/user]
+ & {:keys [parent-run]} :- [:maybe [:map {:closed true}
+ [:parent-run {:optional true} [:maybe [:tuple [:enum :job :dag] ms/PositiveInt]]]]]]
(let [[parent-type parent-id] parent-run]
(try
(transform-run/start-run! id (cond-> {:run_method run-method}
@@ -163,7 +187,7 @@
e))
(throw e))))))
-(defn run-cancelable-transform!
+(mu/defn run-cancelable-transform! :- ::transforms-base.schema/execute-base-result
"Execute a transform with cancellation support and proper error handling.
Computes `source-range-params` once upfront, saves them to the run record, and passes them
@@ -171,7 +195,17 @@
Options:
- `:ex-message-fn` change how caught exceptions are presented to the user in run logs, by default the same as clojure.core/ex-message"
- [run-id transform driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}]
+ [run-id :- ms/PositiveInt
+ transform :- ::transforms-base.schema/transform
+ driver :- :keyword
+ {:keys [db-id conn-spec output-schema]} :- [:map {:closed true}
+ [:db-id ::lib.schema.id/database]
+ [:conn-spec (ms/InstanceOfClass clojure.lang.IPersistentMap)]
+ [:output-schema [:maybe :string]]]
+ run-transform! :- [:=> [:cat ::transforms-base.schema/chan [:maybe ::transforms-base.schema/source-range-params]]
+ ::transforms-base.schema/execute-base-result]
+ & {:keys [ex-message-fn] :or {ex-message-fn ex-message}} :- [:maybe [:map {:closed true}
+ [:ex-message-fn {:optional true} [:=> [:cat (ms/InstanceOfClass Throwable)] [:maybe :string]]]]]]
;; local run is responsible for status, using canceling lifecycle
(let [cancel-chan (a/promise-chan)
transform-timeout (transforms.settings/transform-timeout)
@@ -243,9 +277,22 @@
(transform-run/fail-started-run! run-id {:message (ex-message-fn t)}))
(throw t))))))
-(defn is-temp-transform-table?
+(mu/defn is-temp-transform-table? :- :boolean
"Return true when `table` matches the transform temporary table naming pattern and transforms are enabled."
- [table]
+ [table :- [:or :metabase.warehouse-schema.schema/table [:map {:closed true} [:name :string]]]]
(boolean
(when-let [table-name (and (premium-features/any-transforms-enabled?) (:name table))]
(str/starts-with? (u/lower-case-en table-name) driver.u/transform-temp-table-prefix))))
+
+(mu/defn source-readable? :- :boolean
+ "Whether the current user may read `transform`, saved or not: its kind of transform is enabled, and the user is a
+ superuser or a data analyst who may query its source, looked up in `models-cache` when given."
+ ([transform :- ::transforms.schema/transform.definition]
+ (source-readable? transform nil))
+ ([transform :- ::transforms.schema/transform.definition
+ models-cache :- [:maybe ::source-models]]
+ (boolean
+ (and (check-feature-enabled transform)
+ (or api/*is-superuser?*
+ (and (api/entitled-data-analyst?)
+ (source-tables-readable? transform models-cache)))))))
diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.clj
index ffa53d70df9..e05f0f80e4b 100644
--- a/src/metabase/transforms_base/core.clj
+++ b/src/metabase/transforms_base/core.clj
@@ -12,9 +12,11 @@
[metabase.transforms-base.interface :as transforms-base.i]
;; Load query implementation - registers multimethods
[metabase.transforms-base.query]
- [metabase.transforms-base.util :as transforms-base.u]))
+ [metabase.transforms-base.schema :as transforms-base.schema]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.util.malli :as mu]))
-(defn execute!
+(mu/defn execute! :- ::transforms-base.schema/execute-base-result
"Execute transform and return result map without writing transform_run rows.
This is the main entry point for base transform execution. Use this when you
@@ -37,9 +39,10 @@
Target table sync (metabase_table writes) still occurs - this is necessary
for the table to be usable after execution."
- ([transform]
+ ([transform :- ::transforms-base.schema/transform]
(execute! transform nil))
- ([transform opts]
+ ([transform :- ::transforms-base.schema/transform
+ opts :- [:maybe ::transforms-base.schema/execute-base-options]]
(let [result (transforms-base.i/execute-base! transform opts)]
(when (= :succeeded (:status result))
(transforms-base.u/complete-execution! transform opts))
diff --git a/src/metabase/transforms_base/interface.clj b/src/metabase/transforms_base/interface.clj
index c01a185a115..711e3d44d87 100644
--- a/src/metabase/transforms_base/interface.clj
+++ b/src/metabase/transforms_base/interface.clj
@@ -7,11 +7,13 @@
For scheduled execution with transform_run tracking, see metabase.transforms.interface."
(:require
- [metabase.util.log :as log]))
+ [metabase.transforms-base.schema :as transforms-base.schema]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
-(defn transform->transform-type
+(mu/defn transform->transform-type :- :keyword
"Extract the transform type from a transform's source."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(-> transform :source :type keyword))
(defmulti source-db-id
diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.clj
index 7c14282c353..d0b8e8932b1 100644
--- a/src/metabase/transforms_base/ordering.clj
+++ b/src/metabase/transforms_base/ordering.clj
@@ -9,14 +9,18 @@
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.transforms-base.db :as transforms-base.db]
[metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
- [metabase.util.malli :as mu])
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms])
(:import
(clojure.lang ExceptionInfo)))
@@ -24,10 +28,14 @@
;;; ------------------------------------------------- Query Dependencies -------------------------------------------------
-(defn query-table-dependencies
+(mr/def ::table-dependencies
+ "The dependencies of a transform: computed live as a set, or stored as a sequence."
+ [:or [:set ::transforms-base.schema/table-dependency] [:sequential ::transforms-base.schema/table-dependency]])
+
+(mu/defn query-table-dependencies :- [:set ::transforms-base.schema/table-dependency]
"Compute table dependencies for a query transform.
This is the base implementation - callers may wrap with additional error handling."
- [{:keys [source]}]
+ [{:keys [source]} :- ::transforms-base.schema/transform]
(let [query (-> (:query source)
transforms-base.u/massage-sql-query
qp.preprocess/preprocess)
@@ -41,11 +49,16 @@
{:table table-id}))
(lib/all-source-table-ids query)))))
-(defn- database-routing-error-ex-data [^Throwable e]
+(mu/defn- database-routing-error-ex-data :- [:maybe [:map {:closed true}
+ [:status-code :int]
+ [:database-routing-enabled :boolean]
+ [:database-id [:maybe :int]]
+ [:database-name [:maybe :string]]]]
+ [e :- [:maybe (ms/InstanceOfClass Throwable)]]
(when e
(if (:database-routing-enabled (ex-data e))
(ex-data e)
- (recur (.getCause e)))))
+ (recur (.getCause ^Throwable e)))))
(defmethod transforms-base.i/table-dependencies :query
[transform]
@@ -63,11 +76,11 @@
;;; ------------------------------------------------- Ordering Logic -------------------------------------------------
-(defn stored-or-live-deps
+(mu/defn stored-or-live-deps :- ::table-dependencies
"Dependencies of `transform`, preferring the precomputed `:table_dependencies` column and falling
back to a live `table-dependencies` call when it is absent (`nil`). An empty (but non-nil) stored
value is taken at face value as \"no dependencies\"."
- [transform]
+ [transform :- ::transforms-base.schema/transform-row]
(if-some [deps (:table_dependencies transform)]
deps
(transforms-base.i/table-dependencies
@@ -76,9 +89,9 @@
(and (not (:source transform)) (:id transform))
(assoc :source (transforms-base.db/transform-source (:id transform)))))))
-(defn references-card-or-snippet?
+(mu/defn references-card-or-snippet? :- :boolean
"True if `transform`'s source query reads through a saved card or native snippet."
- [transform]
+ [transform :- ::transforms-base.schema/transform-row]
(let [query (:query (or (:source transform)
(transforms-base.db/transform-source (:id transform))))]
(boolean
@@ -88,12 +101,12 @@
(seq (lib/all-template-tag-snippet-ids query)))
(catch Throwable _ false))))))
-(defn safe-table-dependencies
+(mu/defn safe-table-dependencies :- ::table-dependencies
"Like `stored-or-live-deps`, but returns `#{}` if the computation throws. Used by cycle
detection where a single broken transform must not block the whole check. Callers that need
to know *which* transforms failed should walk the graph themselves and capture the failure
ids — see `transform-ordering`."
- [transform]
+ [transform :- ::transforms-base.schema/transform-row]
(try
(stored-or-live-deps transform)
(catch Throwable _ #{})))
@@ -104,7 +117,7 @@
[:id :metabase.lib.schema.id/transform]
[:target_table_id {:optional true} [:maybe :metabase.lib.schema.id/table]]])
-(mu/defn- output-table-map
+(mu/defn- output-table-map :- [:map-of :metabase.lib.schema.id/table :metabase.lib.schema.id/transform]
[transforms :- [:sequential TransformIdAndTargetTable]]
(into {}
(keep (fn [{:keys [target_table_id id]}]
@@ -112,14 +125,21 @@
[target_table_id id])))
transforms))
-(defn- target-ref-map
+(mr/def ::target-refs
+ "Transform ids by target `[database_id schema table_name]` triple and by target Table id."
+ [:map {:closed true}
+ [:by-triple [:map-of [:tuple [:maybe :int] [:maybe :string] :string] ::lib.schema.id/transform]]
+ [:by-table-id [:map-of ::lib.schema.id/table ::lib.schema.id/transform]]])
+
+(mu/defn- target-ref-map :- ::target-refs
"Build maps for resolving dependencies to transform ids.
Returns {:by-triple {[database_id schema table_name] -> transform_id}
:by-table-id {table_id -> transform_id}}"
- [transforms]
+ [transforms :- [:sequential ::transforms-base.schema/transform-row]]
{:by-triple (into {}
- (map (fn [{:keys [id target]}]
- [[(:database target) (:schema target) (:name target)] id]))
+ (keep (fn [{:keys [id target]}]
+ (when target
+ [[(:database target) (:schema target) (:name target)] id])))
transforms)
:by-table-id (into {}
(keep (fn [{:keys [id target_table_id]}]
@@ -127,10 +147,13 @@
[target_table_id id])))
transforms)})
-(defn- resolve-dependency
+(mu/defn- resolve-dependency :- [:maybe ::lib.schema.id/transform]
"Resolve a single dependency to a transform id, or nil if not resolvable.
Used to map table/transform/table-ref dependencies to actual transform ids."
- [{dep-table :table dep-transform :transform :keys [table-ref]} output-tables transform-ids target-refs]
+ [{dep-table :table dep-transform :transform :keys [table-ref]} :- ::transforms-base.schema/table-dependency
+ output-tables :- [:map-of ::lib.schema.id/table ::lib.schema.id/transform]
+ transform-ids :- [:set ::lib.schema.id/transform]
+ target-refs :- ::target-refs]
(or (get output-tables dep-table)
(get (:by-table-id target-refs) dep-table)
(get transform-ids dep-transform)
@@ -138,7 +161,11 @@
(let [{:keys [database_id schema table]} table-ref]
(get (:by-triple target-refs) [database_id schema table])))))
-(defn transform-ordering
+(mu/defn transform-ordering :- [:map {:closed true}
+ [:dependencies [:map-of ::lib.schema.id/transform [:set ::lib.schema.id/transform]]]
+ [:not-found [:set ::lib.schema.id/transform]]
+ [:failed [:set ::lib.schema.id/transform]]
+ [:uncached [:map-of ::lib.schema.id/transform ::table-dependencies]]]
"Compute the execution ordering for the dependency closure of `start-ids`.
Walks the dependency graph starting from `start-ids`, calling `table-dependencies` only on
@@ -168,7 +195,8 @@
lets the caller persist freshly computed deps so later reads hit the cache instead.
Diagnostics in `:not-found` and `:failed` let the caller decide how to surface them
(logging, metrics, error responses)."
- [start-ids all-transforms]
+ [start-ids :- [:or [:sequential ::lib.schema.id/transform] [:set ::lib.schema.id/transform]]
+ all-transforms :- [:sequential ::transforms-base.schema/transform-row]]
(let [id->xf (u/index-by :id all-transforms)
output-tables (output-table-map (map #(select-keys % [:id :target_table_id]) all-transforms))
target-refs (target-ref-map all-transforms)
@@ -218,14 +246,21 @@
[_id->raw-deps]
nil)
-(defn find-cycle
+(mr/def ::node->children
+ "The children of each node in a graph of transform ids: a map, or a function."
+ [:or
+ [:map-of ::lib.schema.id/transform [:or [:sequential ::lib.schema.id/transform] [:set ::lib.schema.id/transform]]]
+ [:=> [:cat ::lib.schema.id/transform] [:or [:sequential ::lib.schema.id/transform] [:set ::lib.schema.id/transform]]]])
+
+(mu/defn find-cycle :- [:maybe [:vector ::lib.schema.id/transform]]
"Finds a path containing a cycle in the directed graph `node->children`.
Optionally takes a set of starting nodes. If starting nodes are specified, `node->children` can be any
function-equivalent. Without starting nodes, `node->children` must specifically be a map."
- ([node->children]
+ ([node->children :- ::node->children]
(find-cycle node->children (keys node->children)))
- ([node->children starting-nodes]
+ ([node->children :- ::node->children
+ starting-nodes :- [:maybe [:sequential ::lib.schema.id/transform]]]
(loop [stack (into [] (map #(vector % (ordered-set))) starting-nodes)
visited #{}]
(when-let [[node path] (peek stack)]
@@ -243,7 +278,9 @@
(node->children node))]
(recur stack' (conj visited node))))))))
-(defn get-transform-cycle
+(mu/defn get-transform-cycle :- [:maybe [:map {:closed true}
+ [:cycle-str :string]
+ [:cycle [:vector ::lib.schema.id/transform]]]]
"Get a cycle if it exists (otherwise `nil`). Cycle consists of:
```
@@ -251,7 +288,7 @@
:cycle [1 2]}
```
"
- [{transform-id :id :as to-check}]
+ [{transform-id :id :as to-check} :- ::transforms-base.schema/transform]
(let [db-id (get-in to-check [:source :query :database])
;; Recompute the transform under test live — its source or target may have just changed, so the
;; stored deps and `target_table_id` are stale — and pin its source db. Every other transform
@@ -279,11 +316,13 @@
{:cycle-str (str/join " -> " (map id->name cycle))
:cycle cycle})))
-(defn available-transforms
+(mu/defn available-transforms :- [:sequential ::lib.schema.id/transform]
"Given an ordering (see transform-ordering), a set of running transform ids, and a set of completed transform ids,
computes which transforms are currently able to be run. Returns transform ids in the order that they appear in the
ordering map. If you want them returned in a specific order, use a map with ordered keys, e.g., a sorted-map."
- [ordering running complete]
+ [ordering :- [:map-of ::lib.schema.id/transform [:set ::lib.schema.id/transform]]
+ running :- [:set ::lib.schema.id/transform]
+ complete :- [:set ::lib.schema.id/transform]]
(for [[transform-id deps] ordering
:when (and (not (or (running transform-id)
(complete transform-id)))
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
index 7bb8b5abe70..59e0173a132 100644
--- a/src/metabase/transforms_base/query.clj
+++ b/src/metabase/transforms_base/query.clj
@@ -9,7 +9,6 @@
[metabase.indexes.schema :as indexes.schema]
[metabase.lib.schema.common :as schema.common]
[metabase.query-processor.compile :as qp.compile]
- [metabase.transforms-base.db :as transforms-base.db]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
@@ -51,7 +50,15 @@
;;; ------------------------------------------------- Helpers -------------------------------------------------
-(defn- transform-opts [transform-type transform]
+(mu/defn- transform-opts :- [:or
+ [:map {:closed true} [:overwrite? :boolean]]
+ [:map {:closed true}
+ [:merge [:map {:closed true}
+ [:unique-key [:maybe [:sequential :string]]]
+ [:columns [:maybe [:sequential :string]]]]]]
+ [:map {:closed true}]]
+ [transform-type :- [:enum :table :table-incremental]
+ transform :- ::transforms-base.schema/transform]
(case transform-type
:table {:overwrite? true}
:table-incremental
@@ -94,12 +101,7 @@
(when (and cancelled? (cancelled?))
(throw (ex-info "Transform cancelled before start" {:status :cancelled})))
(let [db (get-in source [:query :database])
- {driver :engine :as database} (when (and db (transforms-base.db/database-exists? db))
- (driver.u/lib-database db))
- _ (when-not database
- (throw (ex-info "Source database for this transform has been deleted."
- {:transform-id (:id transform)
- :source-database-id db})))
+ {driver :engine :as database} (driver.u/lib-database db)
_ (transforms-base.u/throw-if-db-routing-enabled! transform database)
;; Full incremental runs (no watermark — either the first ever, or one that follows a
;; checkpoint-config reset) drop and recreate the table rather than appending.
diff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.clj
index 2ebb8814e47..886df326d98 100644
--- a/src/metabase/transforms_base/schema.clj
+++ b/src/metabase/transforms_base/schema.clj
@@ -8,11 +8,14 @@
[metabase.indexes.schema :as indexes.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms])
(:import
(clojure.core.async.impl.channels ManyToManyChannel)
- (clojure.lang Atom)))
+ (clojure.lang Atom)
+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
+ (java.util Date)))
;;; ------------------------------------------------- Primitives -------------------------------------------------
@@ -24,6 +27,35 @@
;;; ------------------------------------------------- Transform -------------------------------------------------
+(mr/def ::temporal
+ "A temporal value: a checkpoint column's value or a run's timestamp."
+ [:or
+ (ms/InstanceOfClass Instant)
+ (ms/InstanceOfClass Date)
+ (ms/InstanceOfClass LocalDate)
+ (ms/InstanceOfClass LocalDateTime)
+ (ms/InstanceOfClass LocalTime)
+ (ms/InstanceOfClass OffsetDateTime)
+ (ms/InstanceOfClass OffsetTime)
+ (ms/InstanceOfClass ZonedDateTime)])
+
+(mr/def ::instantable
+ "A value that converts to an Instant: a temporal or an ISO-8601 string."
+ [:or :string ::temporal])
+
+(mr/def ::column-types
+ "The types of a column, as `lib.types.isa` predicates read them: what a Field row carries before it becomes Lib
+ metadata."
+ [:map {:closed true}
+ [:base-type [:maybe :keyword]]
+ [:effective-type [:maybe :keyword]]])
+
+(mr/def ::lookback
+ "A lookback window on an incremental source: how far behind the checkpoint each run re-reads."
+ [:map {:closed true}
+ [:value pos-int?]
+ [:unit [:or :string :keyword]]])
+
(mr/def ::source-incremental-strategy
"An incremental strategy on a transform's source, as `get-source-range-params` reads it. The full,
`:type`-dispatched shape (checkpoint/append/merge variants) is owned by `metabase.transforms.schema`;
@@ -31,9 +63,7 @@
[:map {:closed true}
[:type {:optional true} [:or :string :keyword]]
[:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]
- [:lookback {:optional true} [:maybe [:map {:closed true}
- [:value pos-int?]
- [:unit [:or :string :keyword]]]]]])
+ [:lookback {:optional true} [:maybe ::lookback]]])
(mr/def ::target-incremental-strategy
"An incremental strategy on a transform's target: append the new rows, or merge them on a unique key."
@@ -46,8 +76,8 @@
(mr/def ::transform-target
"Target specification for a transform. Must include at least :type and :name."
[:map {:closed true}
- [:type :string]
- [:database {:optional true} :int]
+ [:type [:or :string :keyword]]
+ [:database {:optional true} [:maybe :int]]
[:schema {:optional true} [:maybe :string]]
[:name :string]
[:indexes {:optional true} [:sequential ::indexes.schema/index-structured]]
@@ -70,19 +100,46 @@
[:map {:closed true}
[:transform-id {:optional true} pos-int?]]])
+(mr/def ::query-source
+ "The `:source` of a query transform."
+ [:map {:closed true}
+ [:type [:enum :query "query"]]
+ [:query :metabase.lib.schema/query]
+ [:source-incremental-strategy {:optional true} [:maybe ::source-incremental-strategy]]])
+
+(mr/def ::python-source
+ "The `:source` of a Python transform: its script and the tables it reads."
+ [:map {:closed true}
+ [:type [:enum :python "python"]]
+ [:body :string]
+ [:source-tables [:sequential :metabase.transforms-base.util/source-table-entry]]
+ [:source-database {:optional true} [:maybe :int]]
+ [:source-incremental-strategy {:optional true} [:maybe ::source-incremental-strategy]]])
+
+(mr/def ::transform-source
+ "The `:source` of a transform, by its `:type`."
+ [:multi {:dispatch (comp keyword :type)}
+ [:query ::query-source]
+ [:python ::python-source]])
+
+(mr/def ::table-dependency
+ "A logical table dependency of a transform, computed or stored in its `:table_dependencies`; see
+ [[metabase.transforms-base.interface/table-dependencies]]."
+ [:or
+ [:map {:closed true} [:table ::lib.schema.id/table]]
+ [:map {:closed true} [:transform ::lib.schema.id/transform]]
+ [:map {:closed true} [:table-ref [:map {:closed true}
+ [:database_id {:optional true} [:maybe :int]]
+ [:schema {:optional true} [:maybe :string]]
+ [:table :string]]]]])
+
(mr/def ::transform
"A transform map as expected by execute-base! implementations. The full transforms module
(metabase.transforms.schema) hydrates and stores more columns on this same map; they're declared here,
optional, so that richer value can be threaded through the shared execute-base! machinery unchanged."
[:map {:closed true}
+ [:source ::transform-source]
[:id {:optional true} pos-int?]
- [:source [:map {:closed true}
- [:type [:or :string :keyword]]
- [:query {:optional true} [:maybe :metabase.lib-be.schema/maybe-legacy-query]]
- [:body {:optional true} :string]
- [:source-tables {:optional true} [:sequential :metabase.transforms-base.util/source-table-entry]]
- [:source-database {:optional true} :int]
- [:source-incremental-strategy {:optional true} [:maybe ::source-incremental-strategy]]]]
[:target {:optional true} [:maybe ::transform-target]]
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
@@ -99,13 +156,93 @@
[:target_db_id {:optional true} [:maybe ::lib.schema.id/database]]
[:last_checkpoint_value {:optional true} [:maybe :string]]
[:target_table_id {:optional true} [:maybe ::lib.schema.id/table]]
- [:table_dependencies {:optional true} [:maybe [:sequential [:or
- [:map {:closed true} [:table ::lib.schema.id/table]]
- [:map {:closed true} [:transform ::lib.schema.id/transform]]]]]]
- [:tag_ids {:optional true} [:maybe [:sequential pos-int?]]]])
+ [:table_dependencies {:optional true} [:maybe [:sequential ::table-dependency]]]
+ [:tag_ids {:optional true} [:maybe [:sequential pos-int?]]]
+ [:transform_tag_ids {:optional true} [:maybe [:sequential pos-int?]]]
+ [:creator {:optional true} [:maybe [:ref :metabase.users.schema/user]]]
+ [:table {:optional true} [:maybe [:ref :metabase.warehouse-schema.schema/table]]]
+ [:last_run {:optional true} [:maybe [:ref :metabase.transforms.schema/transform-run]]]
+ [:collection {:optional true} [:maybe [:ref :metabase.collections.schema/collection-or-root]]]
+ [:owner {:optional true} [:maybe [:ref :metabase.transforms.schema/transform.owner]]]
+ [:can_read {:optional true} :boolean]
+ [:can_write {:optional true} :boolean]
+ [:can_execute {:optional true} :boolean]])
;;; ----------------------------------------- Source Range Params -----------------------------------------------
+(mr/def ::transform-row
+ "A Transform, or the subset of its columns a caller selected, with the heavy `:source` allowed to be absent."
+ [:map {:closed true}
+ [:source {:optional true} ::transform-source]
+ [:id {:optional true} pos-int?]
+ [:target {:optional true} [:maybe ::transform-target]]
+ [:name {:optional true} :string]
+ [:description {:optional true} [:maybe :string]]
+ [:full-incremental-run? {:optional true} :boolean]
+ [:entity_id {:optional true} [:maybe :string]]
+ [:created_at {:optional true} [:maybe ms/TemporalInstant]]
+ [:updated_at {:optional true} [:maybe ms/TemporalInstant]]
+ [:source_type {:optional true} [:maybe [:or :keyword :string]]]
+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:source_database_id {:optional true} [:maybe ::lib.schema.id/database]]
+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]
+ [:owner_user_id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:owner_email {:optional true} [:maybe :string]]
+ [:target_db_id {:optional true} [:maybe ::lib.schema.id/database]]
+ [:last_checkpoint_value {:optional true} [:maybe :string]]
+ [:target_table_id {:optional true} [:maybe ::lib.schema.id/table]]
+ [:table_dependencies {:optional true} [:maybe [:sequential ::table-dependency]]]
+ [:tag_ids {:optional true} [:maybe [:sequential pos-int?]]]
+ [:transform_tag_ids {:optional true} [:maybe [:sequential pos-int?]]]
+ [:creator {:optional true} [:maybe [:ref :metabase.users.schema/user]]]
+ [:table {:optional true} [:maybe [:ref :metabase.warehouse-schema.schema/table]]]
+ [:last_run {:optional true} [:maybe [:ref :metabase.transforms.schema/transform-run]]]
+ [:collection {:optional true} [:maybe [:ref :metabase.collections.schema/collection-or-root]]]
+ [:owner {:optional true} [:maybe [:ref :metabase.transforms.schema/transform.owner]]]
+ [:can_read {:optional true} :boolean]
+ [:can_write {:optional true} :boolean]
+ [:can_execute {:optional true} :boolean]
+ [:deleted {:optional true} :boolean]])
+
+(defn- run-schema
+ "A run as the transforms module stores or hydrates it, with its timestamps as `timestamp` schemas."
+ [timestamp]
+ (let [timestamps [[:start_time {:optional true} [:maybe timestamp]]
+ [:end_time {:optional true} [:maybe timestamp]]
+ [:last_heartbeat {:optional true} [:maybe ::instantable]]]]
+ [:or
+ [:merge :metabase.transforms.schema/transform-run
+ (into [:map {:closed true}
+ [:transform {:optional true} [:maybe [:ref ::transform-row]]]
+ [:transform_tag_ids {:optional true} [:maybe [:sequential pos-int?]]]]
+ timestamps)]
+ [:merge :metabase.transforms.schema/transform-job-run (into [:map {:closed true}] timestamps)]
+ [:merge :metabase.transforms.schema/transform-dag-run (into [:map {:closed true}] timestamps)]
+ [:map {:closed true}
+ [:run_type [:or :string :keyword]]
+ [:id pos-int?]
+ [:entity_id [:maybe pos-int?]]
+ [:name {:optional true} [:maybe [:or :string mu/localized-string-schema]]]
+ [:entity_name {:optional true} [:maybe :string]]
+ [:direction [:maybe [:or :string :keyword]]]
+ [:transform_count [:maybe :int]]
+ [:run_method [:maybe [:or :string :keyword]]]
+ [:status [:or :string :keyword]]
+ [:is_active [:maybe :boolean]]
+ [:start_time timestamp]
+ [:end_time {:optional true} [:maybe timestamp]]
+ [:message [:maybe :string]]
+ [:user_id [:maybe pos-int?]]]
+ [:map {:closed true} [:start_time timestamp]]]))
+
+(mr/def ::run
+ "A transform run, job run, DAG run, or job's next-run summary, with timestamps as the app DB returns them."
+ (run-schema ::instantable))
+
+(mr/def ::presented-run
+ "A run presented for an API response: its timestamps are ISO-8601 strings."
+ (run-schema :string))
+
(mr/def ::checkpoint-bound
"A bound (lo or hi) for incremental checkpoint filtering."
[:map {:closed true}
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index f1d447563fc..3148a9cc1d6 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -19,6 +19,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
@@ -38,10 +39,13 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
[toucan2.core :as t2])
(:import
+ (clojure.lang Keyword)
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
- (java.util Date)))
+ (java.util Date)
+ (javax.sql DataSource)))
(set! *warn-on-reflection* true)
@@ -53,66 +57,93 @@
;;; ------------------------------------------------- Transform Type Predicates -------------------------------------------------
-(defn- type-is?
+(mr/def ::typed
+ "A transform source, target, or incremental strategy: anything whose `:type` [[type-is?]] reads."
+ [:maybe [:or
+ ::transforms-base.schema/transform-source
+ ::transforms-base.schema/transform-target
+ ::transforms-base.schema/source-incremental-strategy
+ ::transforms-base.schema/target-incremental-strategy]])
+
+(mr/def ::conn-spec
+ "A driver's connection spec: an opaque JDBC spec or data source."
+ [:or
+ [:map {:closed true} [:datasource (ms/InstanceOfClass DataSource)]]
+ ms/DatabaseDetails])
+
+(mr/def ::checkpoint-value
+ "A parsed checkpoint value: a number or a temporal."
+ [:or number? ::transforms-base.schema/temporal])
+
+(mu/defn- type-is? :- :boolean
"True if `m`'s keywordized `:type` equals `t`."
- [m t]
+ [m :- ::typed
+ t :- :keyword]
(= t (keyword (:type m))))
-(defn create-target-schema!
+(mu/defn create-target-schema! :- :nil
"Creates a transform's target `schema` on database `db-id` when it doesn't exist, under a cluster lock per database so concurrent transforms targeting a new schema don't race to create it."
- [driver db-id conn-spec schema]
+ [driver :- :keyword
+ db-id :- ::lib.schema.id/database
+ conn-spec :- ::conn-spec
+ schema :- [:maybe :string]]
(when (and (not (str/blank? schema))
(not (driver/schema-exists? driver db-id schema)))
(cluster-lock/with-cluster-lock (keyword "metabase.transforms-base.util" (str "create-target-schema-" db-id))
(when-not (driver/schema-exists? driver db-id schema)
- (driver/create-schema-if-needed! driver conn-spec schema)))))
+ (driver/create-schema-if-needed! driver conn-spec schema))))
+ nil)
-(defn query-transform?
+(mr/def ::transform-like
+ "A transform row, or the Lib metadata of a transform."
+ [:or ::lib.schema.metadata/transform ::transforms-base.schema/transform])
+
+(mu/defn query-transform? :- :boolean
"Check if this is a query transform: native query / mbql query."
- [transform]
+ [transform :- ::transform-like]
(type-is? (:source transform) :query))
-(defn native-query-transform?
+(mu/defn native-query-transform? :- [:maybe :boolean]
"Check if this is a native query transform.
For query sources, `:query` must be in MBQL 5 format."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(when (query-transform? transform)
(let [query (-> transform :source :query)]
(lib/native-only-query? query))))
-(defn python-transform?
+(mu/defn python-transform? :- :boolean
"Check if this is a Python transform."
- [transform]
+ [transform :- ::transform-like]
(type-is? (:source transform) :python))
-(defn table-target?
+(mu/defn table-target? :- :boolean
"True if `transform` writes to a plain table, recreated on every run."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(type-is? (:target transform) :table))
-(defn incremental-target?
+(mu/defn incremental-target? :- :boolean
"True if `transform` writes to an incremental table."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(type-is? (:target transform) :table-incremental))
-(defn merge-target?
+(mu/defn merge-target? :- :boolean
"True if `transform`'s target uses the merge strategy."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(type-is? (get-in transform [:target :target-incremental-strategy]) :merge))
-(defn checkpoint-source?
+(mu/defn checkpoint-source? :- :boolean
"True if `transform`'s source uses the checkpoint strategy."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(type-is? (get-in transform [:source :source-incremental-strategy]) :checkpoint))
-(defn transform-source-database
+(mu/defn transform-source-database :- [:maybe ::lib.schema.id/database]
"Get the source database from a transform"
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(cond
(query-transform? transform) (-> transform :source :query :database)
(python-transform? transform) (-> transform :source :source-database)))
-(defn full-incremental-run?
+(mu/defn full-incremental-run? :- :boolean
"True when an incremental transform should drop-and-recreate the target rather than append.
Fires when `last_checkpoint_value` is nil (first run, or after the before-update hook clears the watermark on
a `checkpoint-filter-field-id` change), or when pending index changes require rebuilding the table to apply
@@ -121,35 +152,36 @@
The pending-changes check reads the app DB, so execution computes this once
(see [[metabase.transforms.execute]]) and stashes it as `:full-incremental-run?` on the transform; when that
key is present it is returned as-is, keeping the answer stable for the whole run."
- [{:keys [id] :as transform}]
+ [{:keys [id] :as transform} :- ::transforms-base.schema/transform]
(if (contains? transform :full-incremental-run?)
(:full-incremental-run? transform)
(and (incremental-target? transform)
(or (nil? (:last_checkpoint_value transform))
(table-index/pending-changes-for-transform? id)))))
-(defn full-create-run?
+(mu/defn full-create-run? :- :boolean
"True when this run (re)creates the target table -- a plain `:table` run or a full-reset incremental run -- so
index requests are applied and verified."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(or (table-target? transform)
(full-incremental-run? transform)))
;;; ------------------------------------------------- Table Template Tags -------------------------------------------------
-(defn table-template-tag-name
+(mu/defn table-template-tag-name :- [:maybe :string]
"Return the name (key) of the table template tag in `query` that refers to `table-id`, or nil.
This is the table variable the incremental range filter is injected into (see
`inject-filters-into-table-tag`). When `table-id` is nil, any table template tag qualifies."
- [query table-id]
+ [query :- :metabase.lib.schema/query
+ table-id :- [:maybe ::lib.schema.id/table]]
(some (fn [{tag-name :name, :as tag}]
(when (and (#{:table "table"} (:type tag))
(or (nil? table-id) (= table-id (:table-id tag))))
tag-name))
(lib/template-tags query)))
-(defn incremental-table-tag-name
+(mu/defn incremental-table-tag-name :- [:maybe :string]
"Return the name of the table template tag the incremental range filter targets in `transform`, or
nil when the source query has no suitable table variable.
@@ -158,18 +190,18 @@
variable qualifies. A nil result for a table-incremental native transform is a broken state — there
is no table variable to filter on. This can happen when the SQL is edited to drop the table tag
after the transform was made incremental."
- [{:keys [source]}]
+ [{:keys [source]} :- ::transforms-base.schema/transform]
(let [field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
table-id (when field-id (transforms-base.db/field-table-id field-id))]
(table-template-tag-name (:query source) table-id)))
;;; ------------------------------------------------- Transform Normalization -------------------------------------------------
-(defn transform-source-type
+(mu/defn transform-source-type :- [:enum :python :native :mbql]
"Returns the type of a transform's source: :python, :native, or :mbql.
Throws if the source type cannot be detected.
For query sources, `:query` must be in MBQL 5 format."
- [source]
+ [source :- ::transforms-base.schema/transform-source]
(cond
(type-is? source :python) :python
(type-is? source :query) (if (lib/native-only-query? (:query source))
@@ -180,9 +212,9 @@
;;; ------------------------------------------------- Feature Checks -------------------------------------------------
-(defn required-database-features
+(mu/defn required-database-features :- [:sequential :keyword]
"Returns the database features necessary to execute `transform`."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(if (python-transform? transform)
[:transforms/python]
[:transforms/table]))
@@ -201,7 +233,7 @@
(when (driver/table-exists? driver database {:schema default-schema :name (:name table)})
default-schema)))
-(defn qualified-table-name
+(mu/defn qualified-table-name :- :keyword
"Return the transform target as a `:schema/name` (or bare `:name`) HoneySQL
identifier. Consumers downstream rely on `name`/`namespace` to extract the
parts (e.g. `metabase.driver.sql/run-transform! [:sql :table]`), so this stays
@@ -212,15 +244,17 @@
here. It travels separately on `transform-details` as `:output-db` and is
prepended at the SQL emission site. See
[[metabase.driver.sql.query-processor/compile-transform :sql]]."
- [_driver {:keys [schema name]}]
+ [_driver :- :keyword
+ {:keys [schema name]} :- ::transforms-base.schema/transform-target]
(if (str/blank? schema)
(keyword name)
(keyword schema name)))
-(defn temp-table-name
+(mu/defn temp-table-name :- :keyword
"Generate a temporary table name with current timestamp in milliseconds.
If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"
- [driver schema]
+ [driver :- :keyword
+ schema :- [:maybe :string]]
(let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
timestamp (str (System/currentTimeMillis))
prefix (str transform-temp-table-prefix "_")
@@ -234,35 +268,39 @@
;;; ------------------------------------------------- Query Helpers -------------------------------------------------
-(defn massage-sql-query
+(mu/defn massage-sql-query :- :metabase.lib.schema/query
"Adjusts mbql query for use in a transform."
- [query]
+ [query :- :metabase.lib.schema/query]
(-> query
remap/disable-remaps
lib/disable-default-limit))
;;; ------------------------------------------------- Incremental/Checkpoint Helpers -------------------------------------------------
-(defn supported-checkpoint-column?
+(mu/defn supported-checkpoint-column? :- :boolean
"Returns true if `column` (or any map with `:base-type`/`:effective-type`) can be used for
incremental checkpoint filtering.
We support date/datetime and numeric (int/float) columns. Time-only columns are excluded:
their watermarks wrap at midnight, so `>` comparisons against them are meaningless."
- [column]
+ [column :- [:or ::lib.schema.metadata/column ::transforms-base.schema/column-types]]
(or (lib.types.isa/date-or-datetime? column)
(lib.types.isa/numeric? column)))
-(defn- encode-checkpoint-value [v]
+(mu/defn- encode-checkpoint-value :- :string
+ [v :- ::checkpoint-value]
(if (number? v)
(str v)
(u.date/format v)))
-(defn checkpoint-span-attrs
+(mu/defn checkpoint-span-attrs :- [:map {:closed true}
+ [:transform/checkpoint-field-id {:optional true} ::lib.schema.id/field]
+ [:transform/checkpoint-lo {:optional true} :string]
+ [:transform/checkpoint-hi {:optional true} :string]]
"Build a map of OTel span attributes from `source-range-params`. Returns an empty
map when params are nil. Values are encoded as strings (the same encoding used
for persistence) so spans render consistently regardless of base type."
- [source-range-params]
+ [source-range-params :- [:maybe ::transforms-base.schema/source-range-params]]
(let [{:keys [checkpoint-filter-field-id]
{lo-value :value} :lo
{hi-value :value} :hi} source-range-params]
@@ -271,16 +309,18 @@
(some? lo-value) (assoc :transform/checkpoint-lo (encode-checkpoint-value lo-value))
(some? hi-value) (assoc :transform/checkpoint-hi (encode-checkpoint-value hi-value)))))
-(defn save-watermark!
+(mu/defn save-watermark! :- :int
"Commits the incremental transforms :hi watermark value to the appdb."
- [transform-id source-range-params]
+ [transform-id :- ::lib.schema.id/transform
+ source-range-params :- [:maybe ::transforms-base.schema/source-range-params]]
(let [hi-value (:value (:hi source-range-params))]
(transforms-base.db/update-transform! transform-id {:last_checkpoint_value (some-> hi-value encode-checkpoint-value)})))
-(defn save-run-checkpoint-range!
+(mu/defn save-run-checkpoint-range! :- [:maybe :int]
"Persist the checkpoint range (lo/hi) on a transform run record.
This is called early in execution so the range is recorded even if the run fails."
- [run-id source-range-params]
+ [run-id :- [:maybe ms/PositiveInt]
+ source-range-params :- [:maybe ::transforms-base.schema/source-range-params]]
(when (and run-id source-range-params)
(let [{:keys [checkpoint-filter-field-id lo hi]} source-range-params]
(transforms-base.db/update-transform-run! run-id
@@ -288,26 +328,28 @@
lo (assoc :checkpoint_lo_value (encode-checkpoint-value (:value lo)))
hi (assoc :checkpoint_hi_value (encode-checkpoint-value (:value hi))))))))
-(defn- coerce-to-local-datetime
+(mu/defn- coerce-to-local-datetime :- ::transforms-base.schema/temporal
"Coerce a temporal value to LocalDateTime, stripping any timezone information."
- [t]
+ [t :- ::transforms-base.schema/temporal]
(condp instance? t
OffsetDateTime (t/local-date-time t)
ZonedDateTime (t/local-date-time t)
Instant (t/local-date-time t (t/zone-id "UTC"))
t))
-(defn- maybe-coerce-temporal
- [base-type t]
+(mu/defn- maybe-coerce-temporal :- ::checkpoint-value
+ [base-type :- :keyword
+ t :- ::checkpoint-value]
(cond-> t
(and (isa? base-type :type/DateTime)
(not (isa? base-type :type/DateTimeWithTZ)))
coerce-to-local-datetime))
-(defn- parse-checkpoint-value
+(mu/defn- parse-checkpoint-value :- ::checkpoint-value
"Parse a serialized checkpoint value string according to its base-type keyword.
For temporal types, coerces the result to match the column's base-type."
- [base-type s]
+ [base-type :- :keyword
+ s :- [:or :string ::checkpoint-value]]
(cond
(not (string? s)) (maybe-coerce-temporal base-type s)
(isa? base-type :type/Float) (bigdec s)
@@ -318,11 +360,13 @@
(declare ->instant)
-(defn- apply-lookback
+(mu/defn- apply-lookback :- ::checkpoint-value
"Push a checkpoint lower bound back by the lookback window (`value` `unit`s). Only supported
for date and datetime checkpoint columns: time-only columns wrap at midnight, so a window
behind the watermark is meaningless (and day-based units don't apply to them)."
- [checkpoint column {:keys [value unit] :as lookback}]
+ [checkpoint :- ::checkpoint-value
+ column :- ::lib.schema.metadata/column
+ {:keys [value unit] :as lookback} :- ::transforms-base.schema/lookback]
(let [invalid! (fn [msg] (throw (ex-info msg {:transform-message msg, :lookback lookback})))]
(cond
(not (lib.types.isa/date-or-datetime? column))
@@ -337,19 +381,21 @@
:else
(u.date/add checkpoint (keyword unit) (- value)))))
-(defn- checkpoint-compare
+(mu/defn- checkpoint-compare :- :int
"Compare two parsed checkpoint values. Temporal values compare as instants, since the stored
watermark and a fresh QP max can parse to incomparable temporal classes."
- [base-type a b]
+ [base-type :- :keyword
+ a :- ::checkpoint-value
+ b :- ::checkpoint-value]
(if (isa? base-type :type/Temporal)
(compare (->instant a) (->instant b))
(compare a b)))
-(defn- validate-incremental-source!
+(mu/defn- validate-incremental-source! :- :nil
"Throws a user-facing error when an incremental transform can't compute its source range: a
native query without a table variable to inject the filter into, or no checkpoint field
selected."
- [{:keys [source] :as transform}]
+ [{:keys [source] :as transform} :- ::transforms-base.schema/transform]
(when (incremental-target? transform)
(when (and (native-query-transform? transform)
(not (some (fn [tag] (#{:table "table"} (:type tag)))
@@ -362,10 +408,11 @@
"Please select a checkpoint field in the transform settings."))]
(throw (ex-info msg {:transform-message msg}))))))
-(defn- checkpoint-column
+(mu/defn- checkpoint-column :- ::lib.schema.metadata/column
"The checkpoint column fetched from `metadata-provider`, validated to exist, be active, and have
a type supported for incremental filtering."
- [metadata-provider checkpoint-filter-field-id]
+ [metadata-provider :- ::lib.schema.metadata/metadata-provider
+ checkpoint-filter-field-id :- ::lib.schema.id/field]
(let [column (lib.metadata/field metadata-provider checkpoint-filter-field-id)]
(when (or (nil? column) (not (:active column)))
(throw (ex-info "Checkpoint field does not exist or is not active"
@@ -377,14 +424,15 @@
{:column column})))
column))
-(defn- inject-filters-into-table-tag
+(mu/defn- inject-filters-into-table-tag :- :metabase.lib.schema/query
"Inject `:source-filters` into the table template tag matching the checkpoint field's table.
Instead of manually expanding {{tag}} to a subquery SQL string, this adds filter metadata
to the template tag so the QP's existing pipeline handles substitution. The filters specify
field-id-based comparisons (e.g. :> lo, :<= hi) that get rendered as a filtered subquery
in `->replacement-snippet-info`."
- [query source-range-params]
+ [query :- :metabase.lib.schema/query
+ source-range-params :- ::transforms-base.schema/source-range-params]
(let [{:keys [checkpoint-filter-field-id lo hi column]} source-range-params
table-id (:table-id column)
tag-name (table-template-tag-name query table-id)
@@ -466,14 +514,15 @@
:hi (when (some? hi) {:value hi})}
(some? rows-available) (assoc :rows-available rows-available))))))
-(defn preprocess-incremental-query
+(mu/defn preprocess-incremental-query :- :metabase.lib.schema/query
"Add checkpoint filtering to a query for incremental execution.
For native queries, injects `:source-filters` into the table template tag matching the checkpoint
field's table. The QP's substitution pipeline renders these as a filtered subquery.
For MBQL queries, adds filter clauses `WHERE checkpoint_column > lo AND checkpoint_column <= hi`.
Returns the query unchanged when source-range-params is nil."
- [query source-range-params]
+ [query :- :metabase.lib.schema/query
+ source-range-params :- [:maybe ::transforms-base.schema/source-range-params]]
(if source-range-params
(if (lib/native? query)
;; Native: inject source-filters into the table template tag — QP handles the rest
@@ -496,9 +545,10 @@
(catch Exception e
(qp.catch-exceptions/exception-response e)))))
-(defn compile-source
+(mu/defn compile-source :- ::qp.compile/compiled
"Compile the source query of a transform to SQL, applying incremental filtering if required."
- [{:keys [source] :as transform} source-range-params]
+ [{:keys [source] :as transform} :- ::transforms-base.schema/transform
+ source-range-params :- [:maybe ::transforms-base.schema/source-range-params]]
(let [{:keys [query]} source]
(assert (query-transform? transform))
(-> query
@@ -508,16 +558,18 @@
;;; ------------------------------------------------- Target Table Management -------------------------------------------------
-(defn target-table
+(mu/defn target-table :- [:maybe :metabase.warehouse-schema.schema/table]
"Load the `target` table of a transform from the database specified by `database-id`."
- [database-id target & kv-args]
+ [database-id :- [:maybe ::lib.schema.id/database]
+ target :- ::transforms-base.schema/synced-target
+ & kv-args :- [:* [:or :keyword :string :int :boolean]]]
(when database-id
(some-> (apply transforms-base.db/target-table database-id (:schema target) (:name target) kv-args)
(t2/hydrate :db))))
-(defn target-table-exists?
+(mu/defn target-table-exists? :- :boolean
"Test if the target table of a transform already exists."
- [{:keys [target] :as transform}]
+ [{:keys [target] :as transform} :- ::transforms-base.schema/transform]
(let [db-id (transforms-base.i/target-db-id transform)
{driver :engine :as database} (driver.u/lib-database db-id)]
(driver/table-exists? driver database target)))
@@ -573,9 +625,9 @@
;; TODO this should probably be a function in the sync module
(transforms-base.db/update-table! (:id table) {:active false})))
-(defn delete-target-table!
+(mu/defn delete-target-table! :- [:maybe :int]
"Drop a transform's output table and deactivate its app-db Table row."
- [{:keys [id target], :as transform}]
+ [{:keys [id target], :as transform} :- ::transforms-base.schema/transform]
(when target
(let [database-id (transforms-base.i/target-db-id transform)]
(when database-id
@@ -588,9 +640,9 @@
(log/warnf "Skipping drop of transform target %s for transform %d: database %d not found"
(pr-str target) id database-id))))))
-(defn delete-target-table-by-id!
+(mu/defn delete-target-table-by-id! :- [:maybe :int]
"Delete the target table of the transform specified by `transform-id`."
- [transform-id]
+ [transform-id :- ::lib.schema.id/transform]
(delete-target-table! (transforms-base.db/transform transform-id)))
;;; ------------------------------------------------- Table DDL -------------------------------------------------
@@ -631,24 +683,32 @@
(log/infof "Creating table %s with %d columns" table-name (count columns))
(driver/create-table! driver database-id table-name column-definitions opts)))
-(defn drop-table!
+(mu/defn drop-table! :- :nil
"Drop a table in the database."
- [driver database-id table-name]
+ [driver :- :keyword
+ database-id :- ::lib.schema.id/database
+ table-name :- :keyword]
(log/infof "Dropping table %s" table-name)
- (driver/drop-table! driver database-id table-name))
+ (driver/drop-table! driver database-id table-name)
+ nil)
-(defn rename-tables!
+(mu/defn rename-tables! :- :nil
"Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
This is a simpler, composable operation that only handles renaming."
- [driver database-id rename-map]
+ [driver :- :keyword
+ database-id :- ::lib.schema.id/database
+ rename-map :- [:map-of (ms/InstanceOfClass Keyword) (ms/InstanceOfClass Keyword)]]
(log/infof "Renaming tables: %s" (pr-str rename-map))
- (driver/rename-tables! driver database-id rename-map))
+ (driver/rename-tables! driver database-id rename-map)
+ nil)
;;; ------------------------------------------------- Post-Execution Completion -------------------------------------------------
-(defn- mark-index-failed!
+(mu/defn- mark-index-failed! :- [:maybe :int]
"Best-effort: flag the request for `index` (located by its canonical name) failed, with the error message."
- [transform-id index ^Throwable t]
+ [transform-id :- [:maybe ::lib.schema.id/transform]
+ index :- ::indexes.schema/index-structured
+ t :- (ms/InstanceOfClass Throwable)]
(when transform-id
(transforms-base.db/mark-table-index-failed! transform-id (reconcile/index-name index) (ex-message t))))
@@ -675,20 +735,20 @@
(mark-index-failed! transform-id index t)
(throw t))))))))
-(defn apply-target-indexes!
+(mu/defn apply-target-indexes! :- :nil
"Apply the target's standalone indexes, on full-create runs only (a `:table` run or a full-reset incremental run
recreates the table; appends keep the live table and its indexes). Runs before the run is marked succeeded, so a
failure fails the run record."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(when (and (seq (:indexes (:target transform)))
(full-create-run? transform))
(let [database (driver.u/lib-database (transforms-base.i/target-db-id transform))]
(apply-standalone-indexes! database (assoc (:target transform) :transform-id (:id transform))))))
-(defn- apply-index-outcomes!
+(mu/defn- apply-index-outcomes! :- :nil
"Write a [[reconcile/classify-index-outcomes]] result back: drop the removed rows, and move the rest to their new
status (one update per outcome), keeping `error_message` in step with the transition."
- [by-outcome]
+ [by-outcome :- [:map-of [:enum :succeeded :failed :delete-pending :delete-row] [:sequential ::indexes.schema/table-index]]]
(doseq [[status rows] by-outcome]
(if (= :delete-row status)
(transforms-base.db/delete-table-indexes! (map :id rows))
@@ -699,19 +759,24 @@
(= status :failed)
(assoc :error_message "Index was not found on the target table after the transform ran."))))))
-(defn- mark-indexes-unverifiable!
+(mu/defn- mark-indexes-unverifiable! :- :nil
"Fail the still-running `managed` requests with the driver's reason when the warehouse read that verification needs
has failed. Only running rows change, so the run's generic `finally` backstop finds nothing left to relabel."
- [driver managed schema table-name ^Throwable t]
+ [driver :- :keyword
+ managed :- [:sequential ::indexes.schema/table-index]
+ schema :- [:maybe :string]
+ table-name :- :string
+ t :- (ms/InstanceOfClass Throwable)]
(log/warnf "verify-managed-indexes!: could not read indexes for %s.%s: %s" schema table-name (ex-message t))
(table-index/mark-unverified-running-indexes-failed!
(into #{} (map :id) managed)
- (str "Couldn't read the table's indexes to verify this one: " (reconcile/driver-error-message driver t))))
+ (str "Couldn't read the table's indexes to verify this one: " (reconcile/driver-error-message driver t)))
+ nil)
-(defn verify-managed-indexes!
+(mu/defn verify-managed-indexes! :- :nil
"Reconcile each index request against what's physically in the warehouse and set its `:status`.
Only runs on full-create runs (same guard as [[apply-target-indexes!]])."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(when (full-create-run? transform)
(let [target (:target transform)
managed (if (contains? target :index-request-ids)
@@ -728,7 +793,7 @@
(apply-index-outcomes!
(reconcile/classify-index-outcomes managed (reconcile/warehouse-key-set warehouse-indexes)))))))))
-(defn complete-execution!
+(mu/defn complete-execution! :- :nil
"Post-processing steps after a transform has been executed successfully.
Performs:
@@ -740,7 +805,8 @@
`run-cancelable-transform!` should call this AFTER `succeed-started-run!`
to preserve the correct order of operations. (Standalone indexes are applied earlier, by
`apply-target-indexes!`, so a failure there still fails the run.)"
- [transform opts]
+ [transform :- ::transforms-base.schema/transform
+ opts :- [:maybe ::transforms-base.schema/execute-base-options]]
(let [{:keys [target]} transform
{:keys [publish-events?]
:or {publish-events? true}} opts
@@ -766,28 +832,30 @@
:transform-id (:id transform)
:transform-type (keyword (:type target))
:output-schema (:schema target)
- :output-table (qualified-table-name (:engine database) target)}}))))
+ :output-table (qualified-table-name (:engine database) target)}}))
+ nil))
-(defn output-table
+(mu/defn output-table :- [:maybe :metabase.warehouse-schema.schema/table]
"Return the output table created by a transform, looked up via `transform_id`."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(transforms-base.db/table-for-transform (:id transform)))
-(defn merge-target-unique-key
+(mu/defn merge-target-unique-key :- [:maybe [:sequential :string]]
"Physical column names of a transform's `merge` unique key, or nil when the target isn't a merge target."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(when (merge-target? transform)
(not-empty (mapv :name (get-in transform [:target :target-incremental-strategy :unique-key])))))
-(defn target-column-names
+(mu/defn target-column-names :- [:maybe [:sequential :string]]
"Physical column names of a transform's synced target table, ordered by position, or nil."
- [transform]
+ [transform :- ::transforms-base.schema/transform]
(when-let [table (output-table transform)]
(not-empty (transforms-base.db/active-field-names-for-table (:id table)))))
-(defn validate-merge-unique-key!
+(mu/defn validate-merge-unique-key! :- [:maybe [:sequential :string]]
"Throws if any of `unique-key` is not present in `columns`. Returns `unique-key`."
- [unique-key columns]
+ [unique-key :- [:maybe [:sequential :string]]
+ columns :- [:maybe [:sequential :string]]]
(when-let [missing (not-empty (remove (set columns) unique-key))]
(let [msg (i18n/tru "Merge unique key references columns not present in the target: {0}"
(str/join ", " missing))]
@@ -801,11 +869,11 @@
"Maximum number of table refs to query at once to avoid SQL query size limits."
100)
-(defn batch-lookup-table-ids
+(mu/defn batch-lookup-table-ids :- [:maybe [:map-of [:tuple ms/PositiveInt [:maybe :string] :string] ::lib.schema.id/table]]
"Batch lookup table IDs from ref maps. Returns {[db_id schema name] -> table_id}.
Queries the exact conjunction of each [database_id, schema, table] triple rather than
a Cartesian product of all values. Uses chunking to avoid query size limits."
- [refs]
+ [refs :- [:maybe [:sequential ::source-table-entry]]]
(when (seq refs)
(let [unique-refs (distinct (map (juxt :database_id :schema :table) refs))
fetch-batch (fn [batch]
@@ -814,21 +882,21 @@
(transforms-base.db/table-refs-matching batch)))]
(into {} (mapcat fetch-batch) (partition-all batch-lookup-chunk-size unique-refs)))))
-(defn- source-table-ref->key
+(mu/defn- source-table-ref->key :- [:tuple [:maybe :int] [:maybe :string] [:maybe :string]]
"Convert a source table ref map to a lookup key [db_id schema name]."
- [{:keys [database_id schema table]}]
+ [{:keys [database_id schema table]} :- ::source-table-entry]
[database_id schema table])
-(defn- missing-table-id?
+(mu/defn- missing-table-id? :- :boolean
"Returns true if `v` is a source table ref map that needs table_id lookup."
- [v]
+ [v :- ::source-table-entry]
(and (map? v) (nil? (:table_id v))))
(mr/def ::source-table-entry
"A source table entry in the array format. Combines alias with table reference. Callers may supply just
`:table_id` (looked up to fill in `:database_id`/`:schema`/`:table`) or just
`:database_id`/`:schema`/`:table` (looked up to fill in `:table_id`), so only `:alias` is required."
- [:map {:closed true}
+ [:map {:closed true, :decode/normalize lib.schema.common/normalize-map-no-kebab-case}
[:alias :string]
[:database_id {:optional true} [:maybe :int]]
[:schema {:optional true} [:maybe :string]]
@@ -913,22 +981,15 @@
(assoc v :alias (name alias))))
m)))
-(defn normalize-source-tables-structure
- "Converts legacy map format to vec format if needed, otherwise passes through."
- [st]
- (if (map? st)
- (source-tables-map->vec st)
- st))
-
(def keyword-type-dispatch
"Dispatch function for malli :multi schemas that dispatch on `(keyword (:type m))`."
(comp keyword :type))
;;; ------------------------------------------------- Timestamp Helpers -------------------------------------------------
-(defn ->instant
+(mu/defn ->instant :- [:maybe (ms/InstanceOfClass Instant)]
"Convert a temporal value `t` to an Instant in the system timezone."
- ^Instant [t]
+ [t :- [:maybe ::transforms-base.schema/instantable]]
(when t
(condp instance? t
Instant t
@@ -943,68 +1004,87 @@
(throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
{:temporal t})))))
-(defn utc-timestamp-string
+(mu/defn utc-timestamp-string :- :string
"Convert the timestamp `t` to a UTC ISO-8601 string."
- [t]
+ [t :- ::transforms-base.schema/instantable]
(-> t ->instant str))
-(defn localize-run-timestamps
+(mu/defn localize-run-timestamps :- [:maybe ::transforms-base.schema/presented-run]
"Convert the timestamps of a `run` to ISO strings in UTC."
- [run]
+ [run :- [:maybe ::transforms-base.schema/run]]
(-> run
(u/update-some :start_time utc-timestamp-string)
(u/update-some :end_time utc-timestamp-string)))
-(defn present-run
+(mu/defn present-run :- [:maybe ::transforms-base.schema/presented-run]
"Prepare a transform run for an API response: localize its timestamps and drop internal-only fields."
- [run]
+ [run :- [:maybe ::transforms-base.schema/run]]
(-> run
localize-run-timestamps
(dissoc :last_heartbeat)))
;;; ------------------------------------------------- Filter Transforms -------------------------------------------------
-(defn- matching-timestamp?
- [job field-path {:keys [start end]}]
- (when-let [field-instant (->instant (get-in job field-path))]
+(mr/def ::filterable-row
+ "A row the run filters read: a transform, or a transform job with its tags and runs."
+ [:or
+ ::transforms-base.schema/transform
+ [:merge
+ :metabase.transforms.schema/transform-job
+ [:map {:closed true}
+ [:tag_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]
+ [:last_run {:optional true} [:maybe ::transforms-base.schema/run]]
+ [:next_run {:optional true} [:maybe ::transforms-base.schema/run]]]]])
+
+(mu/defn- matching-timestamp? :- [:maybe :boolean]
+ [job :- ::filterable-row
+ field-path :- [:sequential :keyword]
+ {:keys [start end]} :- [:map {:closed true}
+ [:start {:optional true} [:maybe :string]]
+ [:end {:optional true} [:maybe :string]]]]
+ (when-let [^Instant field-instant (->instant (get-in job field-path))]
(let [parse #(-> % u.date/parse ->instant)]
;; logic here is to find when it's not matching and invert this
(not (or (and start (.isBefore field-instant (parse start)))
(and end (.isAfter field-instant (parse end))))))))
-(defn ->date-field-filter-xf
+(mu/defn ->date-field-filter-xf :- ifn?
"Returns an xform for a date filter."
- [field-path filter-value]
+ [field-path :- [:sequential :keyword]
+ filter-value :- [:maybe :string]]
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(filter #(matching-timestamp? % field-path range))
identity))
-(defn ->status-filter-xf
+(mu/defn ->status-filter-xf :- ifn?
"Returns an xform for a transform run status filter."
- [field-path statuses]
+ [field-path :- [:sequential :keyword]
+ statuses :- [:maybe [:sequential [:or :string :keyword]]]]
(if-let [statuses (->> statuses (map keyword) set not-empty)]
(filter #(statuses (get-in % field-path)))
identity))
-(defn ->tag-filter-xf
+(mu/defn ->tag-filter-xf :- ifn?
"Returns an xform for a transform tag filter."
- [field-path tag-ids]
+ [field-path :- [:sequential :keyword]
+ tag-ids :- [:maybe [:sequential :int]]]
(if-let [tag-ids (-> tag-ids set not-empty)]
(filter #(some tag-ids (get-in % field-path)))
identity))
;;; ------------------------------------------------- Misc -------------------------------------------------
-(defn is-temp-transform-table?
+(mu/defn is-temp-transform-table? :- [:maybe :boolean]
"Return true when `table` matches the transform temporary table naming pattern."
- [table]
+ [table :- :metabase.warehouse-schema.schema/table]
(when-let [table-name (:name table)]
(str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))
-(defn throw-if-db-routing-enabled!
+(mu/defn throw-if-db-routing-enabled! :- :nil
"Throws if the database has routing enabled. Call before any driver operations to get a
clear error message rather than a confusing driver-level failure."
- [transform database]
+ [transform :- ::transforms-base.schema/transform
+ database :- ::lib.schema.metadata/database]
(when (database-routing/db-routing-enabled? database)
(throw (ex-info (i18n/tru "Failed to run the transform ({0}) because the database ({1}) has database routing turned on. Running transforms on databases with db routing enabled is not supported."
(:name transform)
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index bbc72de77f3..39c36b41870 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -11,10 +11,13 @@
[metabase.transforms-rest.api.transform-tag]
[metabase.transforms-rest.api.util :as transforms-rest.api.u]
[metabase.transforms-rest.db :as transforms-rest.db]
+ [metabase.transforms-rest.schema :as transforms-rest.schema]
[metabase.transforms.core :as transforms.core]
[metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
+ [metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru LocalizedString]]
+ [metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2])
@@ -30,9 +33,6 @@
(mr/def ::transform-target ::transforms.schema/transform-target)
-(mr/def ::run-trigger
- [:enum "none" "global-schedule"])
-
(def ^:private CreatorResponse
[:map {:closed true}
[:id pos-int?]
@@ -160,7 +160,7 @@
[:last-run-statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
[:tag-ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
[:database-id {:optional true} [:maybe ms/PositiveInt]]]]
- (transforms.core/get-transforms query-params))
+ (map #(update % :last_run transforms-base.u/present-run) (transforms.core/get-transforms query-params)))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -174,15 +174,15 @@
[:description {:optional true} [:maybe :string]]
[:source ::transforms.schema/transform-source]
[:target ::transforms.schema/transform-target]
- [:run_trigger {:optional true} ::run-trigger]
+ [:run_trigger {:optional true} ::transforms.schema/run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
[:collection_id {:optional true} [:maybe ms/PositiveInt]]
[:owner_user_id {:optional true} [:maybe ms/PositiveInt]]
[:owner_email {:optional true} [:maybe :string]]]]
- (transforms.core/check-feature-enabled! body)
+ (transforms.core/check-feature-enabled! (transforms.u/->definition body))
(api/create-check :model/Transform body)
- (transforms.core/check-database-feature body)
- (transforms.core/validate-incremental-column-type! body)
+ (transforms.core/check-database-feature (transforms.u/->definition body))
+ (transforms.core/validate-incremental-column-type! (transforms.u/->definition body))
(api/check (not (transforms-base.u/target-table-exists? body))
409
(deferred-tru "A table with that name already exists."))
@@ -193,7 +193,10 @@
"Get a specific transform."
[{:keys [id]} :- [:map {:closed true}
[:id ms/PositiveInt]]]
- (transforms.core/get-transform id))
+ (let [transform (transforms.core/get-transform id)]
+ (-> transform
+ (u/update-some :last_run transforms-base.u/present-run)
+ (assoc :requestable_indexes (transforms.core/requestable-indexes (transforms.u/->definition transform))))))
(api.macros/defendpoint :get "/:id/dependencies" :- [:sequential TransformResponse]
"Get the dependencies of a specific transform."
@@ -266,17 +269,16 @@
[_route-params
query-params :-
[:map {:closed true}
- [:types {:optional true} [:maybe (ms/QueryVectorOf [:enum "job" "dag" "transform"])]]
+ [:types {:optional true} [:maybe (ms/QueryVectorOf (ms/enum-decode-keyword [:job :dag :transform]))]]
[:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout" "canceled" "canceling"])]]
[:run-methods {:optional true} [:maybe (ms/QueryVectorOf [:enum "manual" "cron"])]]
[:start-time {:optional true} [:maybe ms/NonBlankString]]
[:end-time {:optional true} [:maybe ms/NonBlankString]]
[:transform-ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]
- [:sort-column {:optional true} [:maybe [:enum "start_time" "end_time"]]]
- [:sort-direction {:optional true} [:maybe [:enum "asc" "desc"]]]]]
+ [:sort-column {:optional true} [:maybe (ms/enum-decode-keyword [:start_time :end_time])]]
+ [:sort-direction {:optional true} [:maybe (ms/enum-decode-keyword [:asc :desc])]]]]
(api/check-data-studio-access)
(-> (transforms.core/paged-run-summaries (assoc query-params
- :types (map keyword (:types query-params))
:offset (request/offset)
:limit (request/limit)))
(update :data transforms.core/present-run-summaries)))
@@ -300,7 +302,7 @@
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transforms.schema/transform-source]
[:target {:optional true} ::transforms.schema/transform-target]
- [:run_trigger {:optional true} ::run-trigger]
+ [:run_trigger {:optional true} ::transforms.schema/run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
[:collection_id {:optional true} [:maybe ms/PositiveInt]]
[:owner_user_id {:optional true} [:maybe ms/PositiveInt]]
@@ -343,19 +345,20 @@
(transforms-rest.db/reset-checkpoint! id)
nil)
-(defn- check-feature-and-lock!
+(mu/defn- check-feature-and-lock! :- :nil
"Check that the transform's premium features are enabled and that transforms are not locked by the
trial quota."
- [transform]
- (transforms.core/check-feature-enabled! transform)
- (api/check (not (transforms.core/transform-locked? transform))
+ [transform :- ::transforms.schema/transform]
+ (transforms.core/check-feature-enabled! (transforms.u/->definition transform))
+ (api/check (not (transforms.core/transform-locked? (transforms.u/->definition transform)))
[402 {:message (deferred-tru "Transforms are temporarily locked because the trial quota has been reached.")
- :error-code "metabase_transforms_locked"}]))
+ :error-code "metabase_transforms_locked"}])
+ nil)
-(defn run-transform!
+(mu/defn run-transform! :- ::transforms-rest.schema/async-run-response
"Run a transform. Returns a 202 response with run_id.
The transform must already be fetched and validated."
- [transform]
+ [transform :- ::transforms.schema/transform]
(check-feature-and-lock! transform)
(transforms-rest.api.u/async-run-response
(deferred-tru "Transform run started")
diff --git a/src/metabase/transforms_rest/api/transform_dag_run.clj b/src/metabase/transforms_rest/api/transform_dag_run.clj
index 70c03bc8b4f..8fe6f2e2dc8 100644
--- a/src/metabase/transforms_rest/api/transform_dag_run.clj
+++ b/src/metabase/transforms_rest/api/transform_dag_run.clj
@@ -10,6 +10,8 @@
[metabase.transforms-rest.api.util :as transforms-rest.api.u]
[metabase.transforms-rest.db :as transforms-rest.db]
[metabase.transforms.core :as transforms.core]
+ [metabase.transforms.schema :as transforms.schema]
+ [metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -19,14 +21,16 @@
"The DAG traversal directions a reprocess run can take."
[:upstream :downstream])
-(defn- check-seed-transform
+(mu/defn- check-seed-transform :- :nil
"Check access to a DAG run by applying `check!` (`api/read-check`/`api/write-check`) to its seed
transform. When the seed was deleted (`source_transform_id` nulled out by the FK) there is no
transform left to check against, so fall back to requiring the data-analyst role."
- [check! {:keys [source_transform_id] :as _dag-run}]
+ [check! :- [:=> [:cat [:= :model/Transform] ms/PositiveInt] ::transforms.schema/transform]
+ {:keys [source_transform_id] :as _dag-run} :- ::transforms.schema/transform-dag-run]
(if source_transform_id
(check! :model/Transform source_transform_id)
- (api/check-data-studio-access)))
+ (api/check-data-studio-access))
+ nil)
(api.macros/defendpoint :get "/:run-id/transform-runs" :- [:sequential transforms-rest.api.u/MemberTransformRunResponse]
"Get the transform runs that made up a specific DAG run."
diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.clj
index a547d96d3e5..38a7e9e91e5 100644
--- a/src/metabase/transforms_rest/api/transform_job.clj
+++ b/src/metabase/transforms_rest/api/transform_job.clj
@@ -10,9 +10,12 @@
[metabase.transforms-rest.api.util :as transforms-rest.api.u]
[metabase.transforms-rest.db :as transforms-rest.db]
[metabase.transforms.core :as transforms.core]
+ [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.util :as transforms.u]
[metabase.util.i18n :refer [deferred-tru LocalizedString]]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -20,6 +23,19 @@
(def ^:private ui-display-types [:cron/raw :cron/builder])
+(mr/def ::hydrated-job
+ [:merge
+ ::transforms.schema/transform-job
+ [:map {:closed true}
+ [:tag_ids {:optional true} [:sequential ms/PositiveInt]]
+ [:last_run {:optional true} [:maybe ::transforms.schema/transform-job-run]]]])
+
+(mr/def ::job-with-next-run
+ [:merge
+ ::hydrated-job
+ [:map {:closed true}
+ [:next_run {:optional true} [:map {:closed true} [:start_time :string]]]]])
+
(def ^:private LastRunResponse
"Schema for a job's last run information."
[:map {:closed true}
@@ -250,8 +266,8 @@
(t2/hydrate :tag_ids :last_run)
(update :last_run transforms-base.u/present-run)))
-(defn- add-next-run
- [{id :id :as job}]
+(mu/defn- add-next-run :- ::job-with-next-run
+ [{id :id :as job} :- ::hydrated-job]
(if-let [start-time (-> id transforms.core/existing-trigger :next-fire-time)]
(assoc job :next_run {:start_time (str (transforms-base.u/->instant start-time))})
job))
diff --git a/src/metabase/transforms_rest/api/util.clj b/src/metabase/transforms_rest/api/util.clj
index 7c260baebd4..1189cdbee09 100644
--- a/src/metabase/transforms_rest/api/util.clj
+++ b/src/metabase/transforms_rest/api/util.clj
@@ -1,8 +1,11 @@
(ns metabase.transforms-rest.api.util
"Helpers shared by the transforms REST API namespaces."
(:require
+ [metabase.transforms-rest.schema :as transforms-rest.schema]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]
[ring.util.response :as response]))
(set! *warn-on-reflection* true)
@@ -30,12 +33,14 @@
[:checkpoint_lo_value {:optional true} [:maybe :string]]
[:checkpoint_hi_value {:optional true} [:maybe :string]]])
-(defn async-run-response
+(mu/defn async-run-response :- ::transforms-rest.schema/async-run-response
"Launch `start!` — a fn of a start-promise — on a virtual thread and respond `202` as soon as the
run row exists, without waiting for the run to finish. `start!` must deliver the promise
`[:started run-id]` once the row is created, `nil` when nothing was run, or a Throwable on a
pre-start failure (rethrown). The response body is `{:message message, <id-key> <run-id-or-nil>}`."
- [message id-key start!]
+ [message :- [:or :string mu/localized-string-schema]
+ id-key :- :keyword
+ start! :- [:=> [:cat (ms/InstanceOfClass clojure.lang.IPending)] :nil]]
(let [start-promise (promise)]
(u.jvm/in-virtual-thread*
(try
diff --git a/src/metabase/transforms_rest/schema.clj b/src/metabase/transforms_rest/schema.clj
new file mode 100644
index 00000000000..10ed7425902
--- /dev/null
+++ b/src/metabase/transforms_rest/schema.clj
@@ -0,0 +1,14 @@
+(ns metabase.transforms-rest.schema
+ "Malli schemas shared by the transforms REST API namespaces."
+ (:require
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
+
+(mr/def ::async-run-response
+ "The `202` response of a request that started a run in the background: its body carries a message and the ID of the
+ run it started, when there is one."
+ [:map {:closed true}
+ [:status [:= 202]]
+ [:headers [:map-of :string :string]]
+ [:body [:map-of :keyword [:maybe [:or :string mu/localized-string-schema ms/PositiveInt]]]]])
diff --git a/src/metabase/warehouses/db.clj b/src/metabase/warehouses/db.clj
index c27aec74a2a..aff8eadd211 100644
--- a/src/metabase/warehouses/db.clj
+++ b/src/metabase/warehouses/db.clj
@@ -120,6 +120,12 @@
(t2/query {:delete-from (t2/table-name :model/Card)
:where [:= :database_id database-id]}))
+(mu/defn delete-transforms-for-database! :- :int
+ "Delete the Transforms reading from the Database with `database-id`, each through its own hooks, returning the number
+ deleted."
+ [database-id :- ::lib.schema.id/database]
+ (t2/delete! :model/Transform :source_database_id database-id))
+
(mu/defn disable-uploads-for-all-databases!
"Disable uploads on every Database that has them enabled, returning the number updated."
[]
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index f301374dc6a..b200f2ee196 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -418,6 +418,7 @@
(secret/delete-orphaned-secrets! database)
(delete-database-fields! id)
(warehouses.db/delete-actions-for-database! id)
+ (warehouses.db/delete-transforms-for-database! id)
(->> (eduction
(map t2.realize/realize)
(partition-all 1000)
diff --git a/test/metabase/analytics/stats_test.clj b/test/metabase/analytics/stats_test.clj
index 7e846ddd115..d5a6955fd59 100644
--- a/test/metabase/analytics/stats_test.clj
+++ b/test/metabase/analytics/stats_test.clj
@@ -708,9 +708,10 @@
(#'stats/transform-metrics))))
(testing "with transforms and recent runs"
(mt/with-temp [:model/Transform transform {:target {:database (mt/id)
- :table "test_table"}
+ :type "table"
+ :name "test_table"}
:name "Test SQL transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}}
:model/TransformRun _ {:start_time (t/minus (t/offset-date-time) (t/hours 1))
:transform_id (:id transform)}]
@@ -718,9 +719,10 @@
(#'stats/transform-metrics)))))
(testing "with old transform runs"
(mt/with-temp [:model/Transform transform {:target {:database (mt/id)
- :table "test_table"}
+ :type "table"
+ :name "test_table"}
:name "Test SQL transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}}
:model/TransformRun _ {:start_time (t/minus (t/offset-date-time) (t/hours 25))
:transform_id (:id transform)}]
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index 737d04cd6ab..06e7eb0c2ff 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -4429,3 +4429,39 @@
"Monthly job" false
"Custom job" true}
(job-active)))))))
+
+(deftest transform-source-database-delete-cascade-test
+ (testing "pz7hdr..kq4w2m delete orphan query transforms, then transforms with their source database"
+ (impl/test-migrations ["pz7hdr" "kq4w2m"] [migrate!]
+ (let [user-id (t2/insert-returning-pk! :core_user {:first_name "Transform"
+ :last_name "Owner"
+ :entity_id (u/generate-nano-id)
+ :date_joined :%now
+ :password "password"
+ :password_salt "salt"})
+ db-id (t2/insert-returning-pk! :metabase_database {:name "Transform Source DB"
+ :engine "h2"
+ :created_at :%now
+ :updated_at :%now
+ :details "{}"})
+ insert-transform (fn [source-type source-db-id]
+ (t2/insert-returning-pk! :transform {:name (mt/random-name)
+ :source "{}"
+ :target "{}"
+ :source_type source-type
+ :source_database_id source-db-id
+ :creator_id user-id
+ :entity_id (u/generate-nano-id)
+ :created_at :%now
+ :updated_at :%now}))
+ transform-id (insert-transform "native" db-id)
+ native-orphan-id (insert-transform "native" nil)
+ mbql-orphan-id (insert-transform "mbql" nil)
+ python-id (insert-transform "python" nil)]
+ (migrate!)
+ (is (not (t2/exists? :transform :id native-orphan-id)))
+ (is (not (t2/exists? :transform :id mbql-orphan-id)))
+ (is (t2/exists? :transform :id python-id))
+ (t2/delete! :metabase_database :id db-id)
+ (is (not (t2/exists? :transform :id transform-id)))))))
diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.clj
index 1664a65cbef..ec4157b3294 100644
--- a/test/metabase/driver/sql/transforms_test.clj
+++ b/test/metabase/driver/sql/transforms_test.clj
@@ -267,7 +267,7 @@
:transform-type :table
:conn-spec (driver/connection-spec driver/*driver* (lib.metadata/database (mt/metadata-provider)))
:query (transforms-base.u/compile-source
- {:source {:type "query" :query (venues-source-query)} :target target} nil)
+ {:source {:type :query :query (venues-source-query)} :target target} nil)
:output-schema (:schema target)
:output-table (transforms-base.u/qualified-table-name driver/*driver* target)}]
(driver/run-transform! driver/*driver* details {})
@@ -287,7 +287,7 @@
(transforms.tu/with-transform-cleanup! [target {:type :table :schema schema
:name "ctas_rows_affected_probe"
:database (mt/id)}]
- (let [transform {:source {:type "query" :query (venues-source-query)}
+ (let [transform {:source {:type :query :query (venues-source-query)}
:target target}
details {:db-id (mt/id)
:database (lib.metadata/database (mt/metadata-provider))
@@ -309,7 +309,7 @@
(let [schema (t2/select-one-fn :schema :model/Table :id (mt/id :venues))
written (venues-row-count)]
(transforms.tu/with-transform-cleanup! [target {:type :table :schema schema :name "insert_rows_affected_probe" :database (mt/id)}]
- (let [transform {:source {:type "query" :query (venues-source-query)}
+ (let [transform {:source {:type :query :query (venues-source-query)}
:target target}
details {:db-id (mt/id)
:database (lib.metadata/database (mt/metadata-provider))
diff --git a/test/metabase/indexes/models/table_index_test.clj b/test/metabase/indexes/models/table_index_test.clj
index effed3745ec..6d7f122ea6e 100644
--- a/test/metabase/indexes/models/table_index_test.clj
+++ b/test/metabase/indexes/models/table_index_test.clj
@@ -10,7 +10,7 @@
(defn- temp-transform-spec []
{:name (mt/random-name)
- :source {:type "query" :query (query-test-util/make-query :source-table "venues")}
+ :source {:type :query :query (query-test-util/make-query :source-table "venues")}
:target {:database (mt/id) :type "table" :schema "public" :name (mt/random-name)}})
(deftest roundtrip-and-defaults-test
diff --git a/test/metabase/indexes/serdes_test.clj b/test/metabase/indexes/serdes_test.clj
index bf4f95009f0..68e3fe171f2 100644
--- a/test/metabase/indexes/serdes_test.clj
+++ b/test/metabase/indexes/serdes_test.clj
@@ -10,7 +10,7 @@
(defn- temp-transform-spec []
{:name (mt/random-name)
- :source {:type "query" :query (query-test-util/make-query :source-table "venues")}
+ :source {:type :query :query (query-test-util/make-query :source-table "venues")}
:target {:database (mt/id) :type "table" :schema "public" :name (mt/random-name)}})
(deftest transform-indexes-inlined-test
diff --git a/test/metabase/indexes_rest/api_test.clj b/test/metabase/indexes_rest/api_test.clj
index d39cdcce3dc..72d081a346a 100644
--- a/test/metabase/indexes_rest/api_test.clj
+++ b/test/metabase/indexes_rest/api_test.clj
@@ -21,11 +21,11 @@
non-hosted instances, so a superuser passes the transform read/write checks the index endpoints rely on."
[]
{:name (mt/random-name)
- :source {:type "query" :query (query-test-util/make-query :source-table "venues")}
+ :source {:type :query :query (query-test-util/make-query :source-table "venues")}
:target {:database (mt/id) :type "table" :schema "public" :name (mt/random-name)}})
(defn- temp-incremental-transform-spec []
- (assoc-in (temp-transform-spec) [:target :type] "table-incremental"))
+ (update (temp-transform-spec) :target assoc :type "table-incremental" :target-incremental-strategy {:type "append"}))
(def ^:private btree {:kind "btree" :name "by_cat" :columns [{:name "name"}]})
diff --git a/test/metabase/mcp/v2/tools/content_test.clj b/test/metabase/mcp/v2/tools/content_test.clj
index 638cdd1ee5c..a7cccf95e96 100644
--- a/test/metabase/mcp/v2/tools/content_test.clj
+++ b/test/metabase/mcp/v2/tools/content_test.clj
@@ -516,10 +516,8 @@
(mt/with-temp-env-var-value! [mb-transforms-enabled true]
(mt/with-temp [:model/Transform {id :id} {:name "t1"
:source {:type :query
- :query {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :venues)}}}
- :target {:type :table
+ :query (venues-query)}
+ :target {:type "table"
:schema (t2/select-one-fn :schema :model/Table
:id (mt/id :venues))
:name "t1_out"}}]
diff --git a/test/metabase/metabot/query_export_test.clj b/test/metabase/metabot/query_export_test.clj
index d67f9de8e94..32fb89f471d 100644
--- a/test/metabase/metabot/query_export_test.clj
+++ b/test/metabase/metabot/query_export_test.clj
@@ -14,31 +14,24 @@
(defn- rendered-source
"Return the exported query for a transform, or nil if export is denied."
[query]
- (get-in (query-export/transform-with-exportable-source {:source {:type "query" :query query}})
+ (get-in (query-export/transform-with-exportable-source {:source {:type :query :query query}})
[:source :query]))
+(defn- snippet-query
+ []
+ (-> (lib/native-query (mt/metadata-provider) "SELECT * FROM {{snip}}")
+ (lib/with-template-tags {"snip" {:type :snippet
+ :name "snip"
+ :display-name "snip"
+ :snippet-id Integer/MAX_VALUE
+ :snippet-name "snip"}})))
+
(deftest transform-source-withholds-native-sql-behind-a-later-stage-test
(testing "native SQL is withheld if its permissions cannot be checked, even when an MBQL stage follows it"
- (let [native-stage {:lib/type :mbql.stage/native
- :native "SELECT * FROM {{snip}}"
- :template-tags {"snip" {:type :snippet
- :name "snip"
- :display-name "snip"
- :snippet-id Integer/MAX_VALUE}}}]
- (mt/with-current-user (mt/user->id :rasta)
- (doseq [stages [[native-stage] [native-stage {:lib/type :mbql.stage/mbql}]]]
- (is (nil? (rendered-source {:lib/type :mbql/query :database (mt/id) :stages stages}))
- (str (count stages) " stage(s)")))))))
-
-(deftest transform-source-survives-a-deleted-source-database-test
- (testing "a transform keeps its source after its database is deleted so that an admin can repair it"
- (mt/with-temp [:model/Database {db-id :id} {}]
- (t2/delete! :model/Database :id db-id)
- (mt/with-current-user (mt/user->id :crowberto)
- (is (str/includes? (str (rendered-source {:database db-id
- :type :native
- :native {:query "SELECT 1"}}))
- "SELECT 1"))))))
+ (mt/with-current-user (mt/user->id :rasta)
+ (doseq [query [(snippet-query) (lib/append-stage (snippet-query))]]
+ (is (nil? (rendered-source query))
+ (str (lib/stage-count query) " stage(s)"))))))
(deftest transform-source-withholds-a-sandbox-hidden-field-test
(testing "a source query is withheld if it uses a field hidden by the user's sandbox"
@@ -65,21 +58,16 @@
(deftest transform-source-renders-an-unchecked-source-unresolved-test
(testing "a source query keeps its IDs unresolved if its permissions cannot be checked"
(mt/with-current-user (mt/user->id :rasta)
- (let [query {:lib/type :mbql/query
- :database (mt/id)
- :stages [{:lib/type :mbql.stage/mbql :source-card 13371337}]}]
- (is (:unchecked? (shared.content-store/query-for-export query false)))
- (is (str/includes? (str (rendered-source query)) ":source-card 13371337"))))))
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/card mp card-id))]
+ (t2/delete! :model/Card :id card-id)
+ (is (:unchecked? (shared.content-store/query-for-export query false)))
+ (is (str/includes? (str (rendered-source query)) (str ":source-card " card-id))))))))
(deftest transform-source-withholds-unchecked-legacy-native-sql-test
(testing "native SQL in the legacy format is withheld if its permissions cannot be checked"
- (let [query {:database (mt/id)
- :type :native
- :native {:query "SELECT * FROM {{snip}}"
- :template-tags {"snip" {:type :snippet
- :name "snip"
- :display-name "snip"
- :snippet-id Integer/MAX_VALUE}}}}]
+ (let [query (lib/->legacy-MBQL (snippet-query))]
(mt/with-current-user (mt/user->id :rasta)
(is (:unchecked? (shared.content-store/query-for-export query false)))
(is (nil? (rendered-source query)))))))
diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.clj
index 2248c10658b..8f976046ea7 100644
--- a/test/metabase/metabot/tools/resources_test.clj
+++ b/test/metabase/metabot/tools/resources_test.clj
@@ -832,7 +832,7 @@
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Transform {transform-id :id transform-name :name}
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT * FROM products WHERE category = 'Gadget'")}}]
(testing "fetches transform info"
@@ -855,7 +855,7 @@
(mt/with-premium-features #{:transforms-basic :hosting :advanced-permissions}
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Orders Rollup"
- :source {:type "query"
+ :source {:type :query
:query (lib/query (mt/metadata-provider)
(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]
(mt/with-data-analyst-role! (mt/user->id :rasta)
@@ -889,7 +889,7 @@
:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
:model/Transform {transform-id :id}
{:name "Private source card"
- :source {:type "query"
+ :source {:type :query
:query (lib/query mp (lib.metadata/card mp card-id))}}]
(let [uri (str "metabase://transform/" transform-id)]
(testing "a source card in an unreadable collection prevents reading the transform"
@@ -1044,7 +1044,7 @@
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Permission test transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}}]
(with-redefs [mi/can-read? (constantly false)]
(doseq [uri [(str "metabase://transform/" transform-id)
@@ -1258,7 +1258,7 @@
:model/Transform {tx-other-db :id}
{:name "Other-DB Transform"
:source_database_id db2
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}}]
(testing "/derived for a table in db1 must exclude transforms whose source is in db2"
(let [{:keys [output]} (read-uris
diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.clj
index d724e57735d..c85a9256aec 100644
--- a/test/metabase/metabot/tools/search_test.clj
+++ b/test/metabase/metabot/tools/search_test.clj
@@ -1279,7 +1279,7 @@
(let [mp (lib-be/application-database-metadata-provider db-id)]
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Test Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query mp "SELECT 1")}}]
(let [mock-results [{:id transform-id :type "transform" :name "Test Transform"}
{:id 999 :type "dashboard" :name "Some Dashboard"}]]
diff --git a/test/metabase/revisions/api_test.clj b/test/metabase/revisions/api_test.clj
index 88901a2dc5d..958c68a9901 100644
--- a/test/metabase/revisions/api_test.clj
+++ b/test/metabase/revisions/api_test.clj
@@ -156,16 +156,14 @@
:model/Database {y-db-id :id} {}
:model/Transform {transform-id :id}
{:name "Repointed Transform"
- :source {:type "query"
- :query {:database x-db-id
- :type "native"
- :native {:query "SELECT 2 --comment"}}}}]
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider x-db-id)
+ "SELECT 2 --comment")}}]
(create-transform-revision! transform-id true :crowberto)
(t2/update! :model/Transform :id transform-id
- {:source {:type "query"
- :query {:database y-db-id
- :type "native"
- :native {:query "SELECT 1"}}}})
+ {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider y-db-id)
+ "SELECT 1")}})
(create-transform-revision! transform-id false :crowberto)
(letfn [(contains-x? [revisions]
(letfn [(x-source? [m]
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 676ce94a1f6..1206efe2060 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -552,7 +552,7 @@
;; Note that these refer to the table names, not the search-model names.
{"core_user" #{"action" "collection" "document" "exploration" "measure" "model_index_value" "report_card" "report_dashboard" "segment" "transform"}
"model_index" #{"model_index_value"}
- "metabase_database" #{"action" "measure" "metabase_table" "model_index_value" "report_card" "segment"}
+ "metabase_database" #{"action" "measure" "metabase_table" "model_index_value" "report_card" "segment" "transform"}
"metabase_table" #{"action" "measure" "model_index_value" "report_card" "segment"}
"document" #{"action" "model_index_value" "report_card"}
"exploration" #{"action" "document" "model_index_value" "report_card"}
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index 38a4a239ea8..76c363c8260 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -400,7 +400,7 @@
(fn [_]
{:entity_id (u/generate-nano-id)
:name (str "Test Transform " (u/generate-nano-id))
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (data/metadata-provider) "SELECT 1 as num")}
:target {:type "table"
:name (str "test_table_" (str/replace (u/generate-nano-id) "-" "_"))
diff --git a/test/metabase/transforms/canceling_test.clj b/test/metabase/transforms/canceling_test.clj
index e90754a43fd..42a98f0caa9 100644
--- a/test/metabase/transforms/canceling_test.clj
+++ b/test/metabase/transforms/canceling_test.clj
@@ -10,7 +10,8 @@
[metabase.transforms.models.transform-run-cancelation :as trc]
[toucan2.core :as t2])
(:import
- (java.time OffsetDateTime)))
+ (java.time OffsetDateTime)
+ (org.quartz JobExecutionContext)))
(set! *warn-on-reflection* true)
@@ -83,7 +84,7 @@
(t2/insert! :model/TransformRunCancelation
{:run_id run-id
:time (.minusMinutes (OffsetDateTime/now) 5)})
- (@#'canceling/cancel-old-transform-runs! nil)
+ (@#'canceling/cancel-old-transform-runs! (reify JobExecutionContext))
(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))
(is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))
(is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
@@ -112,7 +113,7 @@
(t2/insert! :model/TransformRunCancelation {:run_id r2 :time old}))
;; Simulate r2 already finished by a concurrent path before the sweep fires.
(t2/update! :model/TransformRun :id r2 {:is_active nil :status :canceled})
- (@#'canceling/cancel-old-transform-runs! nil)
+ (@#'canceling/cancel-old-transform-runs! (reify JobExecutionContext))
(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))
(is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
(is (= :canceled (t2/select-one-fn :status :model/TransformRun :id r1)))))
@@ -125,7 +126,7 @@
(t2/insert! :model/TransformRunCancelation
{:run_id run-id :time (.minusMinutes (OffsetDateTime/now) 5)})
(with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]
- (@#'canceling/cancel-old-transform-runs! nil))
+ (@#'canceling/cancel-old-transform-runs! (reify JobExecutionContext)))
;; Aggregate error: we don't know per-row magnitude after a rollback, just that the sweep failed.
(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
(is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))))
diff --git a/test/metabase/transforms/crud_test.clj b/test/metabase/transforms/crud_test.clj
index 4e92d8b7cc6..11888fb7bde 100644
--- a/test/metabase/transforms/crud_test.clj
+++ b/test/metabase/transforms/crud_test.clj
@@ -1,13 +1,17 @@
(ns metabase.transforms.crud-test
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
[metabase.test :as mt]
[metabase.transforms.crud :as transforms.crud]))
(set! *warn-on-reflection* true)
(defn- lookback-transform [field-id lookback]
- {:source {:type "query"
+ {:name "Lookback transform"
+ :target {:type "table" :name "lookback_target"}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id field-id
:lookback lookback}}})
diff --git a/test/metabase/transforms/index_test.clj b/test/metabase/transforms/index_test.clj
index 96585e5dbc9..5cef5576c24 100644
--- a/test/metabase/transforms/index_test.clj
+++ b/test/metabase/transforms/index_test.clj
@@ -8,6 +8,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.indexes.reconcile :as reconcile]
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
@@ -29,10 +30,16 @@
:query (query-test-util/make-query
{:source-table (t2/select-one-fn :name :model/Table :id (mt/id :transforms_products))})})
+(defn- native-source
+ "A SQL transform source that selects a constant."
+ []
+ {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")})
+
(defn- python-source
"A Python transform source that returns `transforms_products` unchanged (built via `create-table!`)."
[]
- {:type "python"
+ {:type :python
:source-database (mt/id)
:source-tables [(transforms.tu/source-table-entry "transforms_products" (mt/id :transforms_products))]
:body "def transform(transforms_products):\n return transforms_products"})
@@ -239,7 +246,7 @@
(deftest ^:synchronized apply-index-outcomes!-writes-each-bucket-test
(testing "each classified outcome is written back: status, error message, last_executed_at, and row removal"
(mt/with-temp [:model/Transform {tid :id} {:name (mt/random-name)
- :source {:type "query"}
+ :source (native-source)
:source_database_id (mt/id)
:target {:database (mt/id) :type "table"
:schema "public" :name "t"}}
@@ -282,7 +289,7 @@
(deftest ^:synchronized ddl-failure-marks-row-failed-test
(mt/with-temp [:model/Transform {tid :id} {:name (mt/random-name)
- :source {:type "query"}
+ :source (native-source)
:source_database_id (mt/id)
:target {:database (mt/id) :type "table" :schema "public" :name "t"}}
:model/TableIndex {idx-id :id} {:transform_id tid :index_name "boom_idx"
@@ -306,7 +313,7 @@
(deftest ^:synchronized unreadable-warehouse-marks-running-indexes-failed-test
(testing "a failed warehouse read fails the requests this run started, carrying the driver's own reason"
(mt/with-temp [:model/Transform {tid :id} {:name (mt/random-name)
- :source {:type "query"}
+ :source (native-source)
:source_database_id (mt/id)
:target {:database (mt/id) :type "table" :schema "public" :name "t"}}
:model/TableIndex {running-id :id} {:transform_id tid :index_name "running_idx"
@@ -319,7 +326,7 @@
(with-redefs [driver/fetch-table-indexes (fn [& _] (throw (ex-info access-denied-message {})))]
(transforms-base.u/verify-managed-indexes!
{:id tid
- :source {:type "query"}
+ :source (native-source)
:target {:database (mt/id) :type "table" :schema "public" :name "t"}}))
(testing "the running request is failed, with the driver message trimmed of its version tail"
(is (= :failed (t2/select-one-fn :status :model/TableIndex :id running-id)))
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index 7a69494834a..8663aedc8ec 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -28,6 +28,18 @@
(set! *warn-on-reflection* true)
+(defn- run-ctx
+ [parent-run]
+ {:plan []
+ :deps {}
+ :parent-run parent-run
+ :run-gone? (constantly false)
+ :run-method :cron
+ :user-id nil
+ :add-run-activity! (constantly nil)
+ :lanes {:sql 1 :py 1}
+ :timeout-ms 1000})
+
(deftest job-transform-ids-test
(testing "single tag, single transform"
(mt/with-temp [:model/TransformTag tag {:name "tag-a"}
@@ -125,13 +137,15 @@
:id)))
(is (nil? (#'jobs/next-transform sorted-ordering transforms-by-id #{1 2 3})))))
-(def ^:private query-source {:type "query"})
+(defn- query-source
+ []
+ {:type :query :query (lib/native-query (mt/metadata-provider) "SELECT 1")})
(deftest run-query-transform-without-features-test
(mt/with-temporary-raw-setting-values [transforms-enabled "true"]
(mt/with-premium-features #{}
- (let [query-transform {:id 3
- :source query-source
+ (let [query-transform {:id 3 :target {:type "table" :name "t"}
+ :source (query-source)
:name "Test Query Transform"}
run-id 102
run-called? (atom false)]
@@ -139,8 +153,7 @@
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
(reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job run-id])
(promise) query-transform)
(is (empty? (filter (comp #{:warn} :level) (messages)))
"Should not log warnings when feature is enabled")
@@ -149,14 +162,13 @@
(deftest run-query-transform-skipped-hosted-without-basic-feature-test
(mt/with-premium-features #{:hosting}
- (let [query-transform {:id 1
- :source query-source
+ (let [query-transform {:id 1 :target {:type "table" :name "t"}
+ :source (query-source)
:name "Test Query Transform"}
run-id 100]
(mt/with-log-messages-for-level [messages [metabase.transforms.jobs :warn]]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)]
- (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job run-id])
(promise) query-transform)
(is (= 1 (count (messages)))
"Should log exactly one warning")
@@ -168,8 +180,8 @@
(deftest run-query-transform-with-transforms-basic-feature-test
(mt/with-premium-features #{:hosting :transforms-basic}
- (let [query-transform {:id 3
- :source query-source
+ (let [query-transform {:id 3 :target {:type "table" :name "t"}
+ :source (query-source)
:name "Test Query Transform"}
run-id 102
run-called? (atom false)]
@@ -177,8 +189,7 @@
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
(reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job run-id])
(promise) query-transform)
(is (empty? (filter (comp #{:warn} :level) (messages)))
"Should not log warnings when feature is enabled")
@@ -188,8 +199,8 @@
(deftest run-query-transform-skipped-when-transforms-enabled-false-test
(mt/with-temporary-raw-setting-values [transforms-enabled "false"]
(mt/with-premium-features #{:hosting :transforms-basic}
- (let [query-transform {:id 4
- :source query-source
+ (let [query-transform {:id 4 :target {:type "table" :name "t"}
+ :source (query-source)
:name "Disabled Transform"}
run-id 103
run-called? (atom false)]
@@ -197,8 +208,7 @@
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _]
(reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job run-id] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job run-id])
(promise) query-transform)
(is (= 1 (count (messages)))
"Should log exactly one warning")
@@ -224,15 +234,15 @@
(mt/with-premium-features #{:hosting :transforms-basic}
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true}]
(let [transform {:id 7
+ :target {:type "table" :name "t"}
:source_type :native
- :source query-source
+ :source (query-source)
:name "Locked Transform"}
run-called? (atom false)]
(mt/with-log-messages-for-level [messages [metabase.transforms.jobs :warn]]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job 200] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job 200])
(promise) transform)
(is (false? @run-called?)
"execute! must not be called when the meter is locked")
@@ -243,12 +253,11 @@
(testing "scheduled run-transform! runs normally when the meter is not locked"
(mt/with-premium-features #{:hosting :transforms-basic}
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs false}]
- (let [transform {:id 8 :source_type :native :source query-source :name "Unlocked"}
+ (let [transform {:id 8 :target {:type "table" :name "t"} :source_type :native :source (query-source) :name "Unlocked"}
run-called? (atom false)]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job 201] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job 201])
(promise) transform)
(is (true? @run-called?)))))))
(testing "non-metered transform (transform-metered-as → nil) is never blocked by lock state"
@@ -257,12 +266,11 @@
(mt/with-premium-features #{:hosting :transforms-basic}
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true
:transform-advanced-runs true}]
- (let [transform {:id 9 :source_type :native :source query-source :name "Non-metered"}
+ (let [transform {:id 9 :target {:type "table" :name "t"} :source_type :native :source (query-source) :name "Non-metered"}
run-called? (atom false)]
(mt/with-dynamic-fn-redefs [transform-run/running-run-for-transform-id (constantly nil)
transforms.execute/execute! (fn [_ _] (reset! run-called? true))]
- (#'jobs/run-transform! {:parent-run [:job 202] :run-method :scheduled :user-id nil
- :add-run-activity! (constantly nil)}
+ (#'jobs/run-transform! (run-ctx [:job 202])
(promise) transform)
(is (true? @run-called?)
"Non-metered transforms are not gated by lock state")))))))))
@@ -506,10 +514,15 @@
(testing "get-plan with empty transform-ids must not throw on unrelated zombies"
(is (empty? (:order (#'jobs/get-plan #{})))))))))))
+(defn- plan-transform
+ [id]
+ {:id id
+ :source (query-source)})
+
(deftest run-transforms-skips-fresh-pulled-in-deps-test
(mt/with-premium-features #{:transforms-basic}
;; b (2) depends on a (1), plus independent c (3); only b and c are requested, so a is pulled in.
- (let [plan [{:id 1} {:id 2} {:id 3}]
+ (let [plan [(plan-transform 1) (plan-transform 2) (plan-transform 3)]
deps {1 #{} 2 #{1} 3 #{}}
ran (atom #{})]
(testing "a fresh pulled-in dep is not executed, but its dependent still runs"
@@ -519,14 +532,14 @@
#{1})
jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps} {:run-method :cron})))
+ (jobs/run-transforms! 1 #{2 3} {:order plan :deps deps} {:run-method :cron})))
(is (= #{2 3} @ran) "transform 1 (fresh dep) was skipped; 2 and 3 ran")))
(testing "skip-fresh-deps? false (manual run_all) force-refreshes the whole plan"
(reset! ran #{})
(mt/with-dynamic-fn-redefs [freshness/fresh-dep-ids (fn [& _] (throw (ex-info "must not gate when disabled" {})))
jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{2 3} {:order plan :deps deps}
+ (jobs/run-transforms! 1 #{2 3} {:order plan :deps deps}
{:run-method :manual :skip-fresh-deps? false})))
(is (= #{1 2 3} @ran) "with skipping disabled, the dep runs too"))))
(testing "a plan that is entirely skipped exits cleanly as succeeded with nothing run"
@@ -534,7 +547,7 @@
(mt/with-dynamic-fn-redefs [freshness/fresh-dep-ids (fn [_now _dep-ids] #{1})
jobs/run-transform! (fn [_ _ t] (swap! ran conj (:id t)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{} {:order [{:id 1}] :deps {1 #{}}} {:run-method :cron})))
+ (jobs/run-transforms! 1 #{} {:order [(plan-transform 1)] :deps {1 #{}}} {:run-method :cron})))
(is (= #{} @ran)))))))
(deftest run-transforms!-race-condition-test
@@ -616,17 +629,17 @@
(deftest run-transforms-parallel-dispatch-test
(mt/with-premium-features #{:transforms-basic}
(testing "Independent transforms dispatch concurrently up to the configured limit"
- (let [plan [{:id 1} {:id 2} {:id 3} {:id 4}]
+ (let [plan [(plan-transform 1) (plan-transform 2) (plan-transform 3) (plan-transform 4)]
deps {1 #{} 2 #{} 3 #{} 4 #{}}
barrier (CyclicBarrier. 4)]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
(mt/with-dynamic-fn-redefs [jobs/run-transform! (fn [_ _ _]
(.await barrier 5 TimeUnit/SECONDS))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual}))
+ (jobs/run-transforms! 1 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual}))
"all four independents must reach the barrier (i.e. be live simultaneously)")))))
(testing "Dependents wait for their dependencies even when concurrency > 1"
- (let [plan [{:id 1} {:id 2}]
+ (let [plan [(plan-transform 1) (plan-transform 2)]
deps {1 #{} 2 #{1}}
order (atom [])]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
@@ -634,21 +647,21 @@
(when (= 1 (:id transform))
(Thread/sleep 100))
(swap! order conj (:id transform)))]
- (jobs/run-transforms! 0 #{1 2} {:order plan :deps deps} {:run-method :manual})
+ (jobs/run-transforms! 1 #{1 2} {:order plan :deps deps} {:run-method :manual})
(is (= [1 2] @order) "a must complete before b starts")))))
(testing "Dependents of a failed transform are skipped transitively"
- (let [plan [{:id 1} {:id 2} {:id 3}]
+ (let [plan [(plan-transform 1) (plan-transform 2) (plan-transform 3)]
deps {1 #{} 2 #{1} 3 #{2}}]
(mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
(mt/with-dynamic-fn-redefs [jobs/run-transform! (fn [_ _ transform]
(when (= 1 (:id transform))
(throw (ex-info "boom" {}))))]
- (let [result (jobs/run-transforms! 0 #{1 2 3} {:order plan :deps deps} {:run-method :manual})]
+ (let [result (jobs/run-transforms! 1 #{1 2 3} {:order plan :deps deps} {:run-method :manual})]
(is (= :failed (::jobs/status result)))
(is (= #{1 2 3} (set (map (comp :id ::jobs/transform) (::jobs/failures result))))
"a failed, b & c skipped as dep-failed"))))))
(testing "Concurrency is capped by the setting"
- (let [plan (mapv (fn [i] {:id i}) (range 1 9))
+ (let [plan (mapv plan-transform (range 1 9))
deps (into {} (map (fn [i] [i #{}]) (range 1 9)))
;; 2-party barrier: workers must pair up before either can exit. This gives
;; deterministic "two alive" windows we can observe via the live counter — no sleeps.
@@ -661,10 +674,10 @@
(swap! max-live max n))
(.await partner-up 5 TimeUnit/SECONDS)
(swap! live dec))]
- (jobs/run-transforms! 0 (set (range 1 9)) {:order plan :deps deps} {:run-method :manual})
+ (jobs/run-transforms! 1 (set (range 1 9)) {:order plan :deps deps} {:run-method :manual})
(is (= 2 @max-live) "should never exceed the concurrency setting")))))
(testing "Python transforms are serialized in their own lane (n=1)"
- (let [py (fn [id] {:id id :source {:type :python}})
+ (let [py (fn [id] {:id id :source {:type :python :source-tables [] :body "pass"}})
plan [(py 1) (py 2) (py 3) (py 4)]
deps {1 #{} 2 #{} 3 #{} 4 #{}}
live (atom 0)
@@ -677,12 +690,12 @@
;; python worker a window to show up in the counter.
(Thread/sleep 50)
(swap! live dec))]
- (jobs/run-transforms! 0 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual})
+ (jobs/run-transforms! 1 #{1 2 3 4} {:order plan :deps deps} {:run-method :manual})
(is (= 1 @max-live)
"python transforms must run one at a time even with concurrency=4")))))
(testing "SQL and Python lanes run in parallel without contending for slots"
- (let [py (fn [id] {:id id :source {:type :python}})
- sql (fn [id] {:id id})
+ (let [py (fn [id] {:id id :source {:type :python :source-tables [] :body "pass"}})
+ sql plan-transform
plan [(sql 1) (sql 2) (sql 3) (py 4) (py 5)]
deps {1 #{} 2 #{} 3 #{} 4 #{} 5 #{}}
sql-barrier (CyclicBarrier. 3)
@@ -697,7 +710,7 @@
(swap! py-live dec))
(.await sql-barrier 5 TimeUnit/SECONDS)))]
(is (= {::jobs/status :succeeded}
- (jobs/run-transforms! 0 #{1 2 3 4 5} {:order plan :deps deps} {:run-method :manual}))
+ (jobs/run-transforms! 1 #{1 2 3 4 5} {:order plan :deps deps} {:run-method :manual}))
"all three SQL transforms reach the barrier (run in parallel)")
(is (= 1 @max-py) "python lane stays single-slot")))))))
diff --git a/test/metabase/transforms/models/transform_test.clj b/test/metabase/transforms/models/transform_test.clj
index 94fe86d41c1..8bbcfe463df 100644
--- a/test/metabase/transforms/models/transform_test.clj
+++ b/test/metabase/transforms/models/transform_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.events.core :as events]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.interface :as mi]
@@ -12,7 +13,6 @@
[metabase.stale-test :as stale-test]
[metabase.staleness.core :as staleness]
[metabase.test :as mt]
- [metabase.transforms-base.query :as transforms-base.query]
[metabase.transforms.query-test-util :as query-test-util]
[toucan2.core :as t2]))
@@ -22,48 +22,42 @@
(testing "inserting a transform correctly sets the source-database-id column"
(mt/with-temp [:model/Transform transform
{:name "Test Transform"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}}]
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}}]
(is (= (mt/id) (:source_database_id transform)))))
(testing "A transform with no source database has a nil source_database_id"
(mt/with-temp [:model/Transform transform
{:name "Test Transform"
- :source {:type "python"
- :body "print(\"hello\")"}}]
+ :source {:type :python
+ :source-tables []
+ :body "print(\"hello\")"}}]
(is (nil? (:source_database_id transform)))))
(testing "updating a transform recomputes the source-database-id column from the source"
(mt/with-temp [:model/Transform transform
{:name "Test Transform"
- :source {:type "python"
- :body "print(\"hello\")"}}]
+ :source {:type :python
+ :source-tables []
+ :body "print(\"hello\")"}}]
(is (nil? (:source_database_id transform)))
- (t2/update! :model/Transform :id (:id transform) {:source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}})
+ (t2/update! :model/Transform :id (:id transform) {:source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}})
(is (= (mt/id) (t2/select-one-fn :source_database_id :model/Transform :id (:id transform)))))))
-(deftest source-database-deletion-sets-null-test
- (testing "deleting a database referenced by a transform nulls the transform's source_database_id (ON DELETE SET NULL)"
+(deftest source-database-deletion-deletes-transform-test
+ (testing "deleting database deletes transforms reading from it"
(mt/with-temp [:model/Database {db-id :id} {}
:model/Transform transform
{:name "Test Transform"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"}}}}]
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}}]
(is (= db-id (:source_database_id transform)))
(t2/delete! :model/Database :id db-id)
- (is (nil? (t2/select-one-fn :source_database_id :model/Transform :id (:id transform))))
- ;; transform itself still exists
- (is (some? (t2/select-one :model/Transform :id (:id transform)))))))
+ (is (not (t2/exists? :model/Transform :id (:id transform)))))))
(defn- temp-transform-with-index! [thunk]
(mt/with-temp [:model/Transform {transform-id :id}
{:name (mt/random-name)
- :source {:type "query" :query (query-test-util/make-query :source-table "venues")}
+ :source {:type :query :query (query-test-util/make-query :source-table "venues")}
:target {:database (mt/id) :type "table" :schema "public" :name (mt/random-name)}}
:model/TableIndex {index-id :id}
{:transform_id transform-id
@@ -77,7 +71,7 @@
(temp-transform-with-index!
(fn [transform-id index-id]
(t2/update! :model/Transform :id transform-id
- {:source {:type "query" :query (query-test-util/make-query :source-table "checkins")}})
+ {:source {:type :query :query (query-test-util/make-query :source-table "checkins")}})
(is (= :update-pending (t2/select-one-fn :status :model/TableIndex :id index-id))))))
(testing "retargeting to a different table marks the transform's managed indexes for revalidation"
(temp-transform-with-index!
@@ -91,26 +85,6 @@
(t2/update! :model/Transform :id transform-id {:name "renamed"})
(is (= :succeeded (t2/select-one-fn :status :model/TableIndex :id index-id)))))))
-(deftest can-read-write-on-orphaned-transform-test
- (testing "superusers can read/write a transform whose source database has been deleted, data analysts cannot"
- (mt/with-premium-features #{:transforms-basic :hosting}
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Transform transform
- {:name "Orphan Transform"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"}}}}]
- (t2/delete! :model/Database :id db-id)
- (let [reloaded (t2/select-one :model/Transform :id (:id transform))]
- (is (nil? (:source_database_id reloaded)))
- (mt/with-current-user (mt/user->id :crowberto)
- (is (mi/can-read? reloaded))
- (is (mi/can-write? reloaded)))
- (mt/with-current-user (mt/user->id :rasta)
- (is (not (mi/can-read? reloaded)))
- (is (not (mi/can-write? reloaded)))))))))
-
(mt/when-ee-evailable
(deftest native-transform-requires-query-builder-and-native-test
(testing "Writing a native-query transform requires \"Query builder and native\" on the source database, even if the
@@ -120,13 +94,11 @@
:model/PermissionsGroupMembership _ {:user_id (mt/user->id :rasta) :group_id group-id}
:model/Transform native-transform
{:name "Native Transform"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}}
:model/Transform mbql-transform
{:name "MBQL Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/query (mt/metadata-provider)
(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]
(let [native (t2/select-one :model/Transform :id (:id native-transform))
@@ -152,30 +124,11 @@
(mt/with-current-user (mt/user->id :rasta)
(is (mi/can-write? native))))))))))))
-(deftest execute-fails-clearly-when-source-db-deleted-test
- (testing "running a query transform whose source database has been deleted fails with a clear error"
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Transform transform
- {:name "Will be orphaned"
- :source {:type :query
- :query {:database db-id
- :type :native
- :native {:query "SELECT 1"}}}}]
- (t2/delete! :model/Database :id db-id)
- (let [reloaded (t2/select-one :model/Transform :id (:id transform))
- {:keys [status error]} (transforms-base.query/run-query-transform! reloaded nil)]
- (is (= :failed status))
- (is (re-find #"Source database for this transform has been deleted"
- (ex-message error))))))))
-
(deftest no-events-during-deserialization-test
(testing "Transform lifecycle hooks do not publish events during deserialization"
(let [events-published (atom [])
- source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}]
+ source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}]
(with-redefs [events/publish-event! (fn [topic event]
(swap! events-published conj [topic event]))]
(testing "events fire normally for insert/update/delete"
@@ -202,26 +155,20 @@
(testing "table_dependencies is not computed on insert — it is filled lazily on the first read"
(mt/with-temp [:model/Transform {id :id}
{:name "deps insert"
- :source {:type "query"
- :query {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}}}]
+ :source {:type :query
+ :query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]
(is (nil? (stored-deps id)))))
(testing "changing the source invalidates any cached table_dependencies"
(mt/with-temp [:model/Transform {id :id}
{:name "deps update"
- :source {:type "query"
- :query {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}}}}]
+ :source {:type :query
+ :query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]
;; seed a cached value (as the lazy read path would)
(t2/update! :model/Transform :id id {:table_dependencies [{:table (mt/id :orders)}]})
(is (= [{:table (mt/id :orders)}] (stored-deps id)))
(t2/update! :model/Transform :id id
- {:source {:type "query"
- :query {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :people)}}}})
+ {:source {:type :query
+ :query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :people)))}})
(is (nil? (stored-deps id))))))
(defn- insert-run! [transform-id status months-ago]
@@ -356,16 +303,15 @@
(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}
:model/Transform transform
{:name "Reads a Card"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query (format "SELECT * FROM {{#%d}} AS c" card-id)
- :template-tags {(str "#" card-id)
- {:id (str "#" card-id)
- :name (str "#" card-id)
- :display-name (str "#" card-id)
- :type :card
- :card-id card-id}}}}}}]
+ :source {:type :query
+ :query (lib/with-template-tags
+ (lib/native-query (mt/metadata-provider) (format "SELECT * FROM {{#%d}} AS c" card-id))
+ {(str "#" card-id)
+ {:id (str "#" card-id)
+ :name (str "#" card-id)
+ :display-name (str "#" card-id)
+ :type :card
+ :card-id card-id}})}}]
(mt/with-non-admin-groups-no-collection-perms collection
(let [stored (t2/select-one :model/Transform :id (:id transform))
body (into {} stored)]
@@ -392,7 +338,7 @@
:model/PermissionsGroupMembership _ {:user_id (mt/user->id :rasta) :group_id group-id}
:model/Transform transform
{:name "MBQL Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/query (mt/metadata-provider)
(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]
(let [stored (t2/select-one :model/Transform :id (:id transform))]
diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.clj
index 562a8ca5733..d3d3e0bd0fe 100644
--- a/test/metabase/transforms/models_test.clj
+++ b/test/metabase/transforms/models_test.clj
@@ -4,6 +4,8 @@
[clojure.test :refer :all]
[metabase.collections.models.collection :as collection]
[metabase.config.core :as config]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.transforms.models.job-run :as transforms.job-run]
@@ -164,25 +166,19 @@
:name "other_table"}
;; Create transforms targeting these tables
:model/Transform transform1 {:name "Transform with schema"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}
:target {:type "table"
:schema "public"
:name "test_table_with_schema"
- :db_id db-id}}
+ :database db-id}}
:model/Transform transform2 {:name "Transform with NULL schema"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 2"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 2")}
:target {:type "table"
:schema nil
:name "test_table_null_schema"
- :db_id db-id}}]
+ :database db-id}}]
(t2/update! :model/Transform :id (:id transform1) {:target_table_id table-with-schema-id})
(t2/update! :model/Transform :id (:id transform2) {:target_table_id table-null-schema-id})
(let [transform1 (t2/select-one :model/Transform :id (:id transform1))
@@ -225,27 +221,21 @@
:model/Database {db-id :id} {}
:model/Transform transform1 {:name "Transform with creator"
:creator_id user1-id
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}
:target {:type "table"
:schema "public"
:name "test_table"
- :db_id db-id}}
+ :database db-id}}
:model/User {user2-id :id} user2-data
:model/Transform transform2 {:name "Second transform"
:creator_id user2-id
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 2"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 2")}
:target {:type "table"
:schema "public"
:name "test_table2"
- :db_id db-id}}]
+ :database db-id}}]
(testing "Hydrates creator with user details"
(is (=? (assoc user1-data :id user1-id)
(:creator (t2/hydrate transform1 :creator)))))
@@ -316,28 +306,23 @@
(mt/with-premium-features #{:transforms-basic}
(mt/with-temp [:model/Database {db-id :id} {}
:model/Transform {transform-id :id}
- {:source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}
+ {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id 100}}
:target {:type "table-incremental"
:schema "public"
:name "test_incr"
- :db_id db-id}
+ :database db-id
+ :target-incremental-strategy {:type "append"}}
:last_checkpoint_value "42"}]
(testing "checkpoint is present before update"
(let [t (t2/select-one :model/Transform :id transform-id)]
(is (= "42" (:last_checkpoint_value t)))))
(testing "changing checkpoint-filter-field-id resets checkpoint"
(t2/update! :model/Transform :id transform-id
- {:source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}
+ {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id 200}}})
(let [t (t2/select-one :model/Transform :id transform-id)]
@@ -347,11 +332,8 @@
(t2/update! :model/Transform :id transform-id
{:last_checkpoint_value "99"})
(t2/update! :model/Transform :id transform-id
- {:source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 2"
- :template-tags {}}}
+ {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 2")
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id 200}}})
(let [t (t2/select-one :model/Transform :id transform-id)]
@@ -362,18 +344,16 @@
(mt/with-premium-features #{:transforms-basic}
(mt/with-temp [:model/Database {db-id :id} {}
:model/Transform {transform-id :id}
- {:source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}
+ {:source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id 100
:lookback {:value 4 :unit "day"}}}
:target {:type "table-incremental"
:schema "public"
:name "test_lookback"
- :db_id db-id}}]
+ :database db-id
+ :target-incremental-strategy {:type "append"}}}]
(is (= {:value 4 :unit "day"}
(get-in (t2/select-one :model/Transform :id transform-id)
[:source :source-incremental-strategy :lookback])))))))
diff --git a/test/metabase/transforms/revision_test.clj b/test/metabase/transforms/revision_test.clj
index fadd79dc7b7..796c2797fdf 100644
--- a/test/metabase/transforms/revision_test.clj
+++ b/test/metabase/transforms/revision_test.clj
@@ -1,8 +1,11 @@
(ns metabase.transforms.revision-test
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.revisions.impl.transform]
- [metabase.revisions.models.revision :as revision]))
+ [metabase.revisions.models.revision :as revision]
+ [metabase.test :as mt]))
(comment metabase.revisions.impl.transform/keep-me)
@@ -15,9 +18,8 @@
:name "Test Transform"
:description "A test transform"
:source {:type :query
- :query {:database 1
- :type :query
- :query {:source-table 2}}}
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :venues)))}
:target {:type :table
:name "transformed_table"
:schema "public"
diff --git a/test/metabase/transforms/search_test.clj b/test/metabase/transforms/search_test.clj
index 2225f87cf8f..014f9617cc8 100644
--- a/test/metabase/transforms/search_test.clj
+++ b/test/metabase/transforms/search_test.clj
@@ -5,6 +5,7 @@
[java-time.api :as t]
[metabase.app-db.core :as mdb]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.search.appdb.index :as search.index]
[metabase.search.engine :as search.engine]
[metabase.search.ingestion :as search.ingestion]
@@ -61,10 +62,10 @@
(let [now (t/truncate-to (t/offset-date-time) :millis)]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Test transform"
:description "A test transform"
- :source {:type "query"
- :query (mt/native-query {:query "SELECT 1"})}
- :target {:database (mt/id)
- :table "test_table"}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
+ :target {:type "table" :database (mt/id)
+ :name "test_table"}
:created_at now
:updated_at now}]
(let [ingested-transform (ingest-then-fetch! "transform" "Test transform")]
@@ -80,10 +81,11 @@
(let [now (t/truncate-to (t/offset-date-time) :millis)]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Test MBQL transform"
:description "An MBQL test transform"
- :source {:type "query"
- :query (mt/mbql-query venues {:limit 10})}
- :target {:database (mt/id)
- :table "test_mbql_table"}
+ :source {:type :query
+ :query (-> (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :venues)))
+ (lib/limit 10))}
+ :target {:type "table" :database (mt/id)
+ :name "test_mbql_table"}
:created_at now
:updated_at now}]
(let [ingested-transform (ingest-then-fetch! "transform" "Test MBQL transform")]
@@ -99,10 +101,10 @@
(deftest transform-query-ingestion-test
(testing "Contents of SQL and Python transform sources are extracted and indexed for full-text search"
(when (= (mdb/db-type) :postgres)
- (mt/with-temp [:model/Transform _ {:target {:database (mt/id)
- :table "test_table"}
+ (mt/with-temp [:model/Transform _ {:target {:type "table" :database (mt/id)
+ :name "test_table"}
:name "Test SQL transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}}]
(let [ingested-transform (ingest-then-fetch! "transform" "Test SQL transform")
vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
@@ -110,8 +112,9 @@
(is (re-find #"select" vector-value))
(is (re-find #"sql" vector-value))))
(mt/when-ee-evailable
- (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
- :source {:type "python"
+ (mt/with-temp [:model/Transform _ {:target {:type "table" :database (mt/id) :name "python_target"}
+ :source {:type :python
+ :source-tables []
:source-database (mt/id)
:body "import pandas as pd\n"}
:name "Test python transform"}]
@@ -121,11 +124,12 @@
(is (re-find #"import" vector-value))
(is (re-find #"panda" vector-value)))))
(testing "MBQL queries are not indexed in with_native_query_vector"
- (mt/with-temp [:model/Transform _ {:target {:database (mt/id)
- :table "test_mbql_table"}
+ (mt/with-temp [:model/Transform _ {:target {:type "table" :database (mt/id)
+ :name "test_mbql_table"}
:name "Test MBQL transform"
- :source {:type "query"
- :query (mt/mbql-query venues {:limit 10})}}]
+ :source {:type :query
+ :query (-> (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :venues)))
+ (lib/limit 10))}}]
(let [ingested-transform (ingest-then-fetch! "transform" "Test MBQL transform")
vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
(is (string? vector-value))
@@ -156,10 +160,10 @@
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Customer Revenue Analysis"
:description "Analyzes revenue by customer segment"
- :source {:type "query"
- :query (mt/native-query {:query "SELECT customer_id, SUM(revenue) FROM orders GROUP BY customer_id"})}
- :target {:database (mt/id)
- :table "customer_revenue"}}]
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT customer_id, SUM(revenue) FROM orders GROUP BY customer_id")}
+ :target {:type "table" :database (mt/id)
+ :name "customer_revenue"}}]
(ingest! "transform" [:= :this.id transform-id])
(testing "Can search by transform name"
(let [results (search.tu/search-results "Customer Revenue")]
diff --git a/test/metabase/transforms/usage_test.clj b/test/metabase/transforms/usage_test.clj
index b1df2e5d565..7f2a83023c8 100644
--- a/test/metabase/transforms/usage_test.clj
+++ b/test/metabase/transforms/usage_test.clj
@@ -1,6 +1,7 @@
(ns metabase.transforms.usage-test
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
[metabase.transforms.core :as transforms]
@@ -8,6 +9,12 @@
(set! *warn-on-reflection* true)
+(defn- query-source []
+ {:type :query :query (lib/native-query (mt/metadata-provider) "SELECT 1")})
+
+(def ^:private python-source
+ {:type :python :source-tables [] :body "pass"})
+
(defn- with-mocked-routing!
"Call zero-arg `f` with [[premium-features/transform-metered-as]] redefined to return
the given fixed `bucket` string."
@@ -22,7 +29,7 @@
:transform-advanced-runs true}]
(with-mocked-routing! nil
#(is (false? (transforms.usage/transform-locked?
- {:source {:type "query"}})))))))
+ {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))))
(deftest transform-locked?-basic-bucket-test
(testing "basic-bucket transform follows :transform-basic-runs"
@@ -30,16 +37,16 @@
(fn []
(testing "locked"
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true}]
- (is (true? (transforms.usage/transform-locked? {:source {:type "query"}})))))
+ (is (true? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))
(testing "unlocked"
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs false}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "query"}})))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))
(testing "missing → unlocked"
(mt/with-temporary-setting-values [locked-meters {}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "query"}})))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))
(testing "advanced-bucket lock does not bleed into basic"
(mt/with-temporary-setting-values [locked-meters {:transform-advanced-runs true}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "query"}})))))))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))))))
(deftest transform-locked?-advanced-bucket-test
(testing "advanced-bucket transform follows :transform-advanced-runs"
@@ -47,19 +54,19 @@
(fn []
(testing "python locked"
(mt/with-temporary-setting-values [locked-meters {:transform-advanced-runs true}]
- (is (true? (transforms.usage/transform-locked? {:source {:type "python"}})))))
+ (is (true? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source python-source})))))
(testing "python unlocked"
(mt/with-temporary-setting-values [locked-meters {:transform-advanced-runs false}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "python"}})))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source python-source})))))
(testing "basic-bucket lock does not bleed into advanced"
(mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "python"}})))))))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source python-source})))))))))
(deftest transform-locked?-non-transform-meter-ignored-test
(testing "locks on non-transform meters (e.g. :metabase-ai-tokens) do not affect transforms"
(with-mocked-routing! "transform-basic"
#(mt/with-temporary-setting-values [locked-meters {:metabase-ai-tokens true}]
- (is (false? (transforms.usage/transform-locked? {:source {:type "query"}})))))))
+ (is (false? (transforms.usage/transform-locked? {:name "t" :target {:type "table" :name "t"} :source (query-source)})))))))
(deftest transforms-meter-locked?-test
(testing "FE-facing aggregate: locked iff either transforms meter is locked"
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 7278ef21f77..f7d8879c0ee 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -33,6 +33,9 @@
(set! *warn-on-reflection* true)
+(def ^:private base-transform
+ {:name "t" :target {:type "table" :name "t"} :source {:type :python :source-tables [] :body "pass"}})
+
(deftest temp-table-name-test
(testing "temp-table-name generates valid table names respecting driver limits"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -125,16 +128,20 @@
;;; Filter xf tests
;;; ------------------------------------------------------------
+(def ^:private source-column
+ {:lib/type :metadata/column :name "id" :base-type :type/Integer})
+
(deftest ^:parallel matching-timestamp?-test
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
- (let [matching-timestamp? #'transforms-base.u/matching-timestamp?
- field-path [:start_time]
+ (let [job-row (fn [timestamp] {:id 1 :last_run {:id 1 :start_time timestamp}})
+ matching-timestamp? #'transforms-base.u/matching-timestamp?
+ field-path [:last_run :start_time]
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
(testing "with both start and end bounds"
(are [expected timestamp]
- (= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
+ (= expected (matching-timestamp? (job-row timestamp) field-path range-jan-feb))
nil nil ; missing field returns nil
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
false "2023-12-15T12:00:00Z" ; timestamp before range
@@ -144,22 +151,22 @@
(testing "with only start bound"
(are [expected timestamp]
- (= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
+ (= expected (matching-timestamp? (job-row timestamp) field-path range-start-only))
true "2024-01-15T12:00:00Z" ; timestamp after start
true "2024-02-15T12:00:00Z" ; any timestamp after start
false "2023-12-15T12:00:00Z")) ; timestamp before start
(testing "with only end bound"
(are [expected timestamp]
- (= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
+ (= expected (matching-timestamp? (job-row timestamp) field-path range-end-only))
true "2024-01-15T12:00:00Z" ; timestamp before end
true "2023-12-15T12:00:00Z" ; any timestamp before end
false "2024-02-15T12:00:00Z")) ; timestamp after end
(testing "returns nil when field value is missing"
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
- {}
- {:other "value"})))))
+ {:id 1}
+ {:id 1 :name "job"})))))
;;; ------------------------------------------------- Source Table Resolution Tests ----------------------------------
@@ -172,22 +179,22 @@
(is (nil? (transforms-base.u/batch-lookup-table-ids [])))
(is (nil? (transforms-base.u/batch-lookup-table-ids nil))))
(testing "looks up table without schema"
- (let [refs [{:database_id (:id db) :schema nil :table "table_one"}]
+ (let [refs [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]
result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) nil "table_one"] (:id t1)} result))))
(testing "looks up table with schema"
- (let [refs [{:database_id (:id db) :schema "my_schema" :table "table_two"}]
+ (let [refs [{:alias "t" :database_id (:id db) :schema "my_schema" :table "table_two"}]
result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) "my_schema" "table_two"] (:id t2)} result))))
(testing "handles mixed refs with and without schema"
- (let [refs [{:database_id (:id db) :schema nil :table "table_one"}
- {:database_id (:id db) :schema "my_schema" :table "table_two"}]
+ (let [refs [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}
+ {:alias "t" :database_id (:id db) :schema "my_schema" :table "table_two"}]
result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) nil "table_one"] (:id t1)
[(:id db) "my_schema" "table_two"] (:id t2)}
result))))
(testing "returns empty for non-existent table"
- (let [refs [{:database_id (:id db) :schema nil :table "nonexistent"}]
+ (let [refs [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]
result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {} result)))))))
@@ -261,16 +268,18 @@
(binding [api/*current-user-id* (mt/user->id :lucky)]
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :unrestricted
(testing "returns true for query transform when user can read database"
- (let [transform {:source {:type :query
- :query {:database db-id}}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}}]
(is (true? (transforms.u/source-tables-readable? transform)))))
(testing "returns true for python transform when user can read all source tables"
- (let [transform {:source {:type :python
- :source-tables [{:alias "t1" :table_id table-id}]}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :python
+ :body "pass"
+ :source-tables [{:alias "t1" :table_id table-id}]}}]
(is (true? (transforms.u/source-tables-readable? transform)))))
(testing "handles source tables with table_id"
- (let [transform {:source {:type :python
- :source-tables [{:alias "t1" :table_id table-id}]}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :python
+ :body "pass"
+ :source-tables [{:alias "t1" :table_id table-id}]}}]
(is (true? (transforms.u/source-tables-readable? transform))))))))))))
(deftest source-tables-readable-permissions-test
@@ -281,8 +290,8 @@
:model/Table {table1-id :id} {:db_id db-id :name "test_table_1"}
:model/Table {table2-id :id} {:db_id db-id :name "test_table_2"}]
(testing "Query transforms - blocked database access"
- (let [transform {:source {:type :query
- :query {:database db-id}}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}}]
(mt/with-user-in-groups [group {:name "Blocked Group"}
user [group]]
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked
@@ -291,8 +300,9 @@
(is (false? (transforms.u/source-tables-readable? transform))
"User with blocked database access should not be able to read source database")))))))
(testing "Python transforms - blocked database access"
- (let [transform {:source {:type :python
- :source-tables [{:alias "t1" :table_id table1-id}]}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :python
+ :body "pass"
+ :source-tables [{:alias "t1" :table_id table1-id}]}}]
(mt/with-user-in-groups [group {:name "Blocked Group"}
user [group]]
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked
@@ -301,9 +311,10 @@
(is (false? (transforms.u/source-tables-readable? transform))
"User with blocked database access should not be able to read source tables")))))))
(testing "Python transforms - granular access but missing some tables"
- (let [transform {:source {:type :python
- :source-tables [{:alias "t1" :table_id table1-id}
- {:alias "t2" :table_id table2-id}]}}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :python
+ :body "pass"
+ :source-tables [{:alias "t1" :table_id table1-id}
+ {:alias "t2" :table_id table2-id}]}}]
(mt/with-user-in-groups [group {:name "Limited Granular Group"}
user [group]]
;; Block all-users from viewing data at database level, then grant specific access via test group
@@ -328,20 +339,11 @@
{:id tag-name, :name tag-name, :display-name tag-name, :type :table, :table-id table-id})
(defn- native-source
- "A `:query` transform source whose SQL carries `tags`, normalized the way the API and the app DB both
- normalize it."
+ "A `:query` transform source whose SQL carries `tags`."
[db-id sql tags]
{:type :query
- :query (lib-be/normalize-query
- {:database db-id
- :type :native
- :native {:query sql, :template-tags (into {} (map (juxt :name identity)) tags)}})})
-
-(defn- mbql-source
- "A `:query` transform source whose MBQL query is `inner`, normalized the same way."
- [db-id inner]
- {:type :query
- :query (lib-be/normalize-query {:database db-id, :type :query, :query inner})})
+ :query (-> (lib/native-query (lib-be/application-database-metadata-provider db-id) sql)
+ (lib/with-template-tags (into {} (map (juxt :name identity)) tags)))})
;;; `mt/with-current-user` resolves the user's permissions once for the whole body, so each phase below grants or
;;; revokes first and enters it after, rather than mutating permissions inside it.
@@ -352,9 +354,9 @@
:model/Card {card-id :id} {:collection_id (:id collection)
:dataset_query (mt/mbql-query venues)}]
(mt/with-non-admin-groups-no-collection-perms collection
- (let [transform {:source (native-source (mt/id)
- (format "SELECT * FROM {{#%d}} AS c" card-id)
- [(card-tag card-id)])}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source (native-source (mt/id)
+ (format "SELECT * FROM {{#%d}} AS c" card-id)
+ [(card-tag card-id)])}]
(mt/with-user-in-groups [group {:name "transforms"}
user [group]]
(mt/with-data-analyst-role! (:id user)
@@ -372,7 +374,7 @@
(deftest source-tables-readable?-table-tag-test
(testing "a `{{table}}` tag is authorized against that Table, so a per-table permission still applies"
(let [table-id (mt/id :users)
- transform {:source (native-source (mt/id) "SELECT * FROM {{users}}" [(table-tag "users" table-id)])}]
+ transform {:name "t" :target {:type "table" :name "t"} :source (native-source (mt/id) "SELECT * FROM {{users}}" [(table-tag "users" table-id)])}]
(mt/with-user-in-groups [group {:name "transforms"}
user [group]]
(mt/with-data-analyst-role! (:id user)
@@ -392,14 +394,15 @@
(let [users (mt/id :users)
venues (mt/id :venues)
cats (mt/id :categories)
- plain {:source (mbql-source (mt/id) {:source-table users})}
- joined {:source (mbql-source (mt/id)
- {:source-table venues
- :joins [{:source-table cats
- :alias "c"
- :condition [:= [:field (mt/id :venues :category_id) nil]
- [:field (mt/id :categories :id)
- {:join-alias "c"}]]}]})}]
+ mp (mt/metadata-provider)
+ plain {:name "t" :target {:type "table" :name "t"} :source {:type :query
+ :query (lib/query mp (lib.metadata/table mp users))}}
+ joined {:name "t" :target {:type "table" :name "t"} :source {:type :query
+ :query (-> (lib/query mp (lib.metadata/table mp venues))
+ (lib/join (lib/join-clause
+ (lib.metadata/table mp cats)
+ [(lib/= (lib.metadata/field mp (mt/id :venues :category_id))
+ (lib.metadata/field mp (mt/id :categories :id)))])))}}]
(mt/with-user-in-groups [group {:name "transforms"}
user [group]]
(mt/with-no-data-perms-for-all-users!
@@ -420,7 +423,7 @@
(deftest source-query-permissions-check-requires-user-test
(testing "the source query permission check refuses to run without a bound user"
- (let [transform {:source (native-source (mt/id) "SELECT 1" [])}]
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source (native-source (mt/id) "SELECT 1" [])}]
(binding [api/*current-user-id* nil]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"needs to be run with a bound user"
(transforms.u/check-source-query-permissions! transform)))))))
@@ -459,7 +462,7 @@
:schema nil
:name "some_nil_schema_table"
:active true}]
- (let [found (transforms-base.u/target-table db-id {:schema nil :name "some_nil_schema_table"})]
+ (let [found (transforms-base.u/target-table db-id {:type "table" :schema nil :name "some_nil_schema_table"})]
(is (some? found)
"target-table must find a Table row with schema=nil — otherwise schemas-less drivers (MySQL) would silently miss their targets")
(is (= "some_nil_schema_table" (:name found)))))))
@@ -474,7 +477,7 @@
:name "test_output_table"}
:model/Transform {transform-id :id :as transform}
{:target target
- :source {:type "query"
+ :source {:type :query
:query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]
;; Mock execute-base! to return success without actually running a query,
;; run-cancelable-transform! to bypass schema creation / cancellation infra,
@@ -613,9 +616,10 @@
(mt/with-premium-features #{:transforms-basic}
(mt/with-temporary-setting-values [transform-timeout 90]
(transforms.u/run-cancelable-transform!
- 1 {:id 1} :h2 {:db-id 1 :conn-spec nil :output-schema "x"}
+ 1 (merge base-transform {:id 1 :target {:type "table" :name "t"}}) :h2 {:db-id 1 :conn-spec {} :output-schema "x"}
(fn [_cancel-chan _range-params]
- (reset! driver-observed-timeout-ms driver.settings/*query-timeout-ms*))))))
+ (reset! driver-observed-timeout-ms driver.settings/*query-timeout-ms*)
+ {:status :succeeded})))))
(is (= (u/minutes->ms 90) @driver-observed-timeout-ms)))))
;;; -------------------------------------------------- `:metabase-transforms/incremental-rows` --------------------------------------------------
@@ -636,7 +640,7 @@
transform-run/succeed-started-run! (constantly nil)]
(mt/with-premium-features #{:transforms-basic}
(transforms.u/run-cancelable-transform!
- 1 transform driver {:db-id 1 :conn-spec nil :output-schema "x"}
+ 1 transform driver {:db-id 1 :conn-spec {} :output-schema "x"}
(fn [_cancel-chan _range-params]
{:status :succeeded :result driver-result}))))))
@@ -645,7 +649,7 @@
(testing "Non-incremental transform: `get-source-range-params` returns nil → no emission, both branches stay at zero"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table"}}
+ (merge base-transform {:id 1 :target {:type "table" :name "t"}})
nil
{:rows-affected 100})
(is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
@@ -659,8 +663,8 @@
(testing "First incremental run (no watermark) → {full-incremental-run=true}, both types bumped from the same scan"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}
- {:checkpoint-filter-field-id 42 :rows-available 1000}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value nil})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 1000}
{:rows-affected 1000})
(is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "true"}))))
@@ -673,8 +677,8 @@
(testing "Subsequent incremental run (watermark present) → {full-incremental-run=false}, attrition surfaces as sum mismatch"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
- {:checkpoint-filter-field-id 42 :rows-available 500}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value "42"})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 500}
{:rows-affected 120})
(is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "false"}))))
@@ -687,8 +691,8 @@
(testing "Driver result missing :rows-affected (defensive contract) → no emission on either side"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
- {:checkpoint-filter-field-id 42 :rows-available 999}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value "42"})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 999}
{:some-other-shape "no rows-affected here"})
(is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "false"}))))
@@ -697,8 +701,8 @@
(testing "Python-shaped driver result (clj-http response augmented with :rows-affected by run-python-transform-impl!) emits the metric"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
- {:checkpoint-filter-field-id 42 :rows-available 800}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value "42"})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 800}
{:status 200 :body {:exit_code 0} :rows-affected 750})
(is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "false"}))))
@@ -711,8 +715,8 @@
(testing "Incremental run whose source-range-params lacks :rows-available → no emission on either side"
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
- {:checkpoint-filter-field-id 42}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value "42"})
+ {:column source-column :checkpoint-filter-field-id 42}
{:rows-affected 100})
(is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "false"}))))
@@ -723,8 +727,8 @@
(try
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental" :schema "x" :name "tgt"} :last_checkpoint_value nil}
- {:checkpoint-filter-field-id 42 :rows-available 1000}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :schema "x" :name "tgt"} :last_checkpoint_value nil})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 1000}
{:rows-affected 0})
(is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
{:type "available" :full-incremental-run "true"})))
@@ -739,8 +743,8 @@
(try
(analytics/clear! :metabase-transforms/incremental-rows)
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
- {:checkpoint-filter-field-id 42 :rows-available 500}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value "42"})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 500}
{:rows-affected 120})
(is (== 120 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
{:type "processed" :full-incremental-run "false"})))
@@ -753,14 +757,14 @@
(fn [& _] (throw (ex-info "Synthetic emission failure" {})))]
(is (= {:status :succeeded :result {:rows-affected 1000}}
(run-cancelable-with-mocks!
- {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}
- {:checkpoint-filter-field-id 42 :rows-available 1000}
+ (merge base-transform {:id 1 :target {:type "table-incremental" :name "t"} :last_checkpoint_value nil})
+ {:column source-column :checkpoint-filter-field-id 42 :rows-available 1000}
{:rows-affected 1000}))
"run-cancelable-transform! returns the success envelope; the emission throw is swallowed.")))))
(deftest ^:parallel massage-sql-query-test
(testing "massage-sql-query sets disable-remaps? and disable-max-results?"
- (let [query {:database 1, :type :query, :query {:source-table 1}}
+ (let [query (lib/query (mt/metadata-provider) (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))
massaged (transforms-base.u/massage-sql-query query)]
(is (true? (get-in massaged [:middleware :disable-remaps?])))
(is (true? (get-in massaged [:middleware :disable-max-results?]))))))
@@ -769,8 +773,8 @@
(testing "compile-source produces SQL without a LIMIT clause"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms-basic}
- (let [transform {:source {:type "query"
- :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
+ (let [transform {:name "t" :target {:type "table" :name "t"} :source {:type :query
+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
{:keys [query]} (transforms-base.u/compile-source transform (transforms-base.u/get-source-range-params transform))]
(is (string? query))
(is (not (re-find #"(?i)\bLIMIT\b" query))
diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.clj
index 5d7b54b37ee..9f8be8d2b07 100644
--- a/test/metabase/transforms_base/ordering_test.clj
+++ b/test/metabase/transforms_base/ordering_test.clj
@@ -1,6 +1,9 @@
(ns ^:mb/driver-tests metabase.transforms-base.ordering-test
(:require
[clojure.test :refer :all]
+ [medley.core :as m]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.compile :as qp.compile]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
@@ -12,6 +15,20 @@
(defn- default-schema-or-public []
(or (:default_schema (mt/db)) "public"))
+(defn- table-query [table-id]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp table-id))))
+
+(defn- native-query [sql]
+ (lib/native-query (mt/metadata-provider) sql))
+
+(defn- orders-products-query []
+ (let [mp (mt/metadata-provider)]
+ (lib/join (table-query (mt/id :orders))
+ (lib/join-clause (lib.metadata/table mp (mt/id :products))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))
+ (lib.metadata/field mp (mt/id :products :id)))]))))
+
(defn- make-transform [query & [name schema]]
(let [name (or name (mt/random-name))
schema (or schema (default-schema-or-public))]
@@ -20,7 +37,7 @@
:name (str "transform_" name)
:target {:schema schema
:name name
- :type :table}}))
+ :type "table"}}))
(use-fixtures :once (fn [thunk]
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -28,9 +45,7 @@
(deftest basic-ordering-test
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}})]
+ (table-query (mt/id :orders)))]
(is (= {t1 #{}}
(:dependencies (ordering/transform-ordering #{t1} (t2/select :model/Transform :id t1)))))))
@@ -40,15 +55,11 @@
:model/Field _ {:table_id table
:name "foo"}
:model/Transform {parent :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}}
+ (table-query (mt/id :orders))
"orders_2")
(assoc :target_table_id table))
:model/Transform {child :id} (make-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table table}}
+ (table-query table)
"orders_3")]
(is (= {parent #{}
child #{parent}}
@@ -62,30 +73,22 @@
(mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})
(testing "dependencies are correctly identified when no transforms have been run yet"
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM orders LIMIT 100"}}
+ (native-query "SELECT * FROM orders LIMIT 100")
"orders_transform")
:model/Transform {t2 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM products LIMIT 100"}}
+ (native-query "SELECT * FROM products LIMIT 100")
"products_transform")
:model/Transform {t3 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM orders_transform ot
+ (native-query "SELECT * FROM orders_transform ot
LEFT JOIN products p
ON ot.product_id = p.id
- LIMIT 100"}}
+ LIMIT 100")
"orders_transform_products")
:model/Transform {t4 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM orders_transform ot
+ (native-query "SELECT * FROM orders_transform ot
LEFT JOIN products_transform pt
ON ot.product_id = pt.id
- LIMIT 100"}}
+ LIMIT 100")
"orders_transform_products_transform")]
(is (= #{{:transform t1} {:transform t2}}
(transform-deps-for-db (t2/select-one :model/Transform :id t4))))
@@ -96,22 +99,16 @@
(:dependencies (ordering/transform-ordering #{t1 t2 t3 t4} (t2/select :model/Transform :id [:in [t1 t2 t3 t4]])))))))
(testing "dependencies are correctly identified when some transform have been run and some haven't"
(mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM checkins LIMIT 100"}}
+ (native-query "SELECT * FROM checkins LIMIT 100")
"checkins_transform")
:model/Transform {t2 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM venues LIMIT 100"}}
+ (native-query "SELECT * FROM venues LIMIT 100")
"venues_transform")
:model/Transform {t3 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query "SELECT * FROM venues_transform vt
+ (native-query "SELECT * FROM venues_transform vt
LEFT JOIN checkins_transform ct
ON vt.id = ct.venue_id
- LIMIT 100"}}
+ LIMIT 100")
"venues_transform_2")]
(try
(transforms.execute/execute! transform1 {:run-method :manual})
@@ -130,24 +127,18 @@
(let [[dataset] (sql.tx/qualified-name-components :bigquery-cloud-sdk nil)]
(testing "dependencies are correctly identified when some transform have been run and some haven't"
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset)}}
+ (native-query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset))
"checkins_transform" dataset)
:model/Table {table1 :id} {:schema dataset
:name "checkins_transform"}
:model/Transform {t2 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset)}}
+ (native-query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset))
"venues_transform" dataset)
:model/Transform {t3 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query (format "SELECT * FROM `%s.venues_transform` vt
+ (native-query (format "SELECT * FROM `%s.venues_transform` vt
LEFT JOIN `%s.checkins_transform` ct
ON vt.id = ct.venue_id
- LIMIT 100" dataset dataset)}}
+ LIMIT 100" dataset dataset))
"venues_transform_2" dataset)]
(is (= #{{:table table1} {:transform t2}}
(transform-deps-for-db (t2/select-one :model/Transform :id t3))))
@@ -158,77 +149,36 @@
(deftest ^:parallel basic-dependencies-test
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}})]
+ (table-query (mt/id :orders)))]
(is (= #{{:table (mt/id :orders)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
(deftest ^:parallel joined-dependencies-test
(mt/test-drivers (mt/normal-drivers-with-feature :left-join)
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders),
- :joins
- [{:fields "all",
- :strategy "left-join",
- :alias "Products",
- :condition
- [:=
- [:field
- (mt/id :orders :product_id)
- {:base-type "type/Integer"}]
- [:field
- (mt/id :products :id)
- {:base-type "type/Integer",
- :join-alias "Products"}]],
- :source-table (mt/id :products)}]},
- :parameters []})]
+ (orders-products-query))]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
(deftest card-dependencies-test
(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :left-join]})
- (mt/with-temp [:model/Card {card :id} {:dataset_query {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders),
- :joins
- [{:fields "all",
- :strategy "left-join",
- :alias "Products",
- :condition
- [:=
- [:field
- (mt/id :orders :product_id)
- {:base-type "type/Integer"}]
- [:field
- (mt/id :products :id)
- {:base-type "type/Integer",
- :join-alias "Products"}]],
- :source-table (mt/id :products)}]},
- :parameters []}}
+ (mt/with-temp [:model/Card {card :id} {:dataset_query (orders-products-query)}
:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query
- {:aggregation [["count"]],
- :breakout [["field" "Products__category" {:base-type "type/Text"}]],
- :source-table (str "card__" card)},
- :parameters []})]
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/card mp card))
+ category (m/find-first #(= (mt/id :products :category) (:id %))
+ (lib/breakoutable-columns query))]
+ (-> query
+ (lib/aggregate (lib/count))
+ (lib/breakout category))))]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
(deftest native-dependencies-test
(mt/with-temp [:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type :native
- :native (qp.compile/compile
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}})})]
+ (native-query (:query (qp.compile/compile (table-query (mt/id :orders))))))]
(is (= #{{:table (mt/id :orders)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
@@ -236,37 +186,14 @@
(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table
:native-parameter-card-reference
:left-join]})
- (mt/with-temp [:model/Card {card :id} {:dataset_query {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders),
- :joins
- [{:fields "all",
- :strategy "left-join",
- :alias "Products",
- :condition
- [:=
- [:field
- (mt/id :orders :product_id)
- {:base-type "type/Integer"}]
- [:field
- (mt/id :products :id)
- {:base-type "type/Integer",
- :join-alias "Products"}]],
- :source-table (mt/id :products)}]},
- :parameters []}}
+ (mt/with-temp [:model/Card {card :id} {:dataset_query (orders-products-query)}
:model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type :native,
- :native {:query
- (mt/native-query-with-card-template-tag
- (:engine (mt/db))
- (str "#" card)),
- :template-tags
- {(keyword (str "#" card))
- {:type "card"
- :name "card"
- :display-name "card"
- :card-id card}}}})]
+ (-> (native-query (mt/native-query-with-card-template-tag (:engine (mt/db)) (str "#" card)))
+ (lib/with-template-tags {(str "#" card) {:type :card
+ :name (str "#" card)
+ :display-name (str "#" card)
+ :card-id card
+ :id (str (random-uuid))}})))]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
@@ -298,9 +225,7 @@
:model/Field _ {:table_id table1
:name "foo"}
:model/Transform {t1 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
+ (table-query table1)
"table_1")
(assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_1"
@@ -316,15 +241,11 @@
:model/Field _ {:table_id table2
:name "foo"}
:model/Transform {t1 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
+ (table-query table1)
"table_2")
(assoc :target_table_id table2))
:model/Transform {t2 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table2}}
+ (table-query table2)
"table_1")
(assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_2 -> transform_table_1",
@@ -346,21 +267,15 @@
:model/Field _ {:table_id table3
:name "foo"}
:model/Transform {t1 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
+ (table-query table1)
"table_2")
(assoc :target_table_id table2))
:model/Transform {t2 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table2}}
+ (table-query table2)
"table_3")
(assoc :target_table_id table3))
:model/Transform {t3 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table3}}
+ (table-query table3)
"table_1")
(assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
@@ -374,9 +289,7 @@
:name "foo"}
:model/Table {table2 :id} {:schema (default-schema-or-public), :name "table_2"}
:model/Transform {t1 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
+ (table-query table1)
"table_2")
(assoc :target_table_id table2))]
(let [transform (t2/select-one :model/Transform :id t1)]
@@ -396,15 +309,11 @@
:name "foo"}
:model/Table {table3 :id} {:schema (default-schema-or-public), :name "table_3"}
:model/Transform _ (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
+ (table-query table1)
"table_2")
(assoc :target_table_id table2))
:model/Transform {t2 :id} (-> (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table2}}
+ (table-query table2)
"table_3")
(assoc :target_table_id table3))]
(let [transform (t2/select-one :model/Transform :id t2)]
@@ -424,8 +333,10 @@
(testing "an empty (non-nil) stored value means \"no dependencies\""
(is (= [] (ordering/stored-or-live-deps {:table_dependencies []}))))
(testing "falls back to a live computation when the stored value is nil"
- ;; no :source and an unknown type -> the :default table-dependencies impl returns #{}
- (is (= #{} (ordering/stored-or-live-deps {:id 1})))))
+ (mt/with-temp [:model/Transform {id :id} (make-transform (table-query (mt/id :orders)))]
+ (is (= #{{:table (mt/id :orders)}}
+ (mt/with-metadata-provider (mt/id)
+ (ordering/stored-or-live-deps {:id id})))))))
(deftest transform-ordering-reads-stored-deps-test
(testing "transform-ordering resolves dependencies from :table_dependencies, never parsing SQL"
@@ -434,7 +345,7 @@
(is (= {1 #{} 2 #{1}}
(:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))
(testing "a :table-ref dependency resolves to the producing transform by (db, schema, name)"
- (let [producer {:id 1 :target {:database 10 :schema "public" :name "out"} :table_dependencies []}
+ (let [producer {:id 1 :target {:type "table" :database 10 :schema "public" :name "out"} :table_dependencies []}
consumer {:id 2 :table_dependencies [{:table-ref {:database_id 10 :schema "public" :table "out"}}]}]
(is (= {1 #{} 2 #{1}}
(:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))
@@ -446,9 +357,7 @@
(testing "a transform with no cached :table_dependencies is reported in :uncached and persisted"
(mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})
(mt/with-temp [:model/Transform {id :id} (make-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table (mt/id :orders)}})]
+ (table-query (mt/id :orders)))]
;; simulate a row created before the column existed
(t2/update! (t2/table-name :model/Transform) :id id {:table_dependencies nil})
(mt/with-metadata-provider (mt/id)
diff --git a/test/metabase/transforms_base/ordering_unit_test.clj b/test/metabase/transforms_base/ordering_unit_test.clj
index ee2f93c544a..1ffe19e5b19 100644
--- a/test/metabase/transforms_base/ordering_unit_test.clj
+++ b/test/metabase/transforms_base/ordering_unit_test.clj
@@ -18,15 +18,22 @@
[id dep-ids]
{:id id
:target_table_id (* 100 id)
- :target {:database 1 :schema "public" :name (str "t" id)}
- :test-deps (into #{} (map (fn [dep-id] {:table (* 100 dep-id)})) dep-ids)})
+ :target {:type "table" :database 1 :schema "public" :name (str "t" id)}
+ :source {:type :python
+ :body "pass"
+ :source-tables (mapv (fn [dep-id] {:alias (str "t" dep-id) :table_id (* 100 dep-id)}) dep-ids)}})
+
+(defn- test-deps
+ "The table dependencies a synthetic transform built by [[tx]] declares through its source tables."
+ [transform]
+ (into #{} (map (fn [{:keys [table_id]}] {:table table_id})) (get-in transform [:source :source-tables])))
(deftest transform-ordering-closure-walk-test
(testing "transform-ordering only visits transforms reachable from start-ids"
- ;; Stub table-dependencies to read the :test-deps key we put on the synthetic transforms.
+ ;; Stub table-dependencies to read the source tables we put on the synthetic transforms.
;; This keeps the test focused on the closure-walk and dep-resolution logic without
;; needing a real query processor.
- (with-redefs [transforms-base.i/table-dependencies :test-deps]
+ (with-redefs [transforms-base.i/table-dependencies test-deps]
(testing "empty start-ids returns empty ordering"
(is (=? {:dependencies {} :not-found #{} :failed #{}}
(ordering/transform-ordering #{} [(tx 1 #{})]))))
@@ -61,7 +68,7 @@
(fn [transform]
(if (= (:id transform) 1)
(throw (ex-info "simulated extraction failure" {}))
- (:test-deps transform)))]
+ (test-deps transform)))]
;; Transform 1 is tagged but its extractor throws. The walk catches, captures 1 in :failed,
;; treats 1 as a leaf, and the supposed downstream dep (2) is never visited.
(is (=? {:dependencies {1 #{}} :not-found #{} :failed #{1}}
@@ -71,7 +78,7 @@
(fn [transform]
(if (= (:id transform) 1)
(throw (ex-info "simulated upstream failure" {}))
- (:test-deps transform)))]
+ (test-deps transform)))]
;; Transform 2 is tagged and depends on 1. 2's extractor succeeds, so the edge 2→1 is
;; recorded. Then 1 is visited, its extractor throws, so 1 is captured in :failed and
;; treated as a leaf. The parent edge 2→1 is preserved, which is what run-transforms!
@@ -81,12 +88,11 @@
(deftest passthrough-card-or-snippet-test
(testing "transforms that read through a card/snippet are omitted from :uncached (they pass through)"
- (with-redefs [transforms-base.i/table-dependencies :test-deps
+ (with-redefs [transforms-base.i/table-dependencies test-deps
;; pretend transform 1 references a card/snippet
ordering/references-card-or-snippet? #(= 1 (:id %))]
(let [{:keys [uncached]} (ordering/transform-ordering
#{1 2}
- [{:id 1 :test-deps #{}}
- {:id 2 :test-deps #{}}])]
+ [(tx 1 #{}) (tx 2 #{})])]
(is (not (contains? uncached 1)))
(is (contains? uncached 2))))))
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index d619b26e3ab..a384649e9cc 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -31,28 +31,36 @@
;;; ------------------------------------------------- Merge target helpers -------------------------------------------------
+(defn- transform-with-target
+ [target]
+ {:source {:type :python :body "pass" :source-tables []}
+ :target (merge {:name "target_table"} target)})
+
(deftest merge-target?-test
(testing "true only for a table-incremental target with a merge strategy"
(is (transforms-base.u/merge-target?
- {:target {:type "table-incremental"
- :target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}}))
+ (transform-with-target {:type "table-incremental"
+ :target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}})))
(is (not (transforms-base.u/merge-target?
- {:target {:type "table-incremental"
- :target-incremental-strategy {:type "append"}}})))
- (is (not (transforms-base.u/merge-target? {:target {:type "table"}})))))
+ (transform-with-target {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}}))))
+ (is (not (transforms-base.u/merge-target? (transform-with-target {:type "table"}))))))
(deftest merge-target-unique-key-test
(testing "returns the physical column names of the merge key"
(is (= ["id"]
(transforms-base.u/merge-target-unique-key
- {:target {:target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}})))
+ (transform-with-target {:type "table-incremental"
+ :target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}))))
(is (= ["order_id" "region"]
(transforms-base.u/merge-target-unique-key
- {:target {:target-incremental-strategy
- {:type "merge" :unique-key [{:name "order_id"} {:name "region"}]}}}))))
+ (transform-with-target {:type "table-incremental"
+ :target-incremental-strategy
+ {:type "merge" :unique-key [{:name "order_id"} {:name "region"}]}})))))
(testing "nil when the target isn't a merge target"
(is (nil? (transforms-base.u/merge-target-unique-key
- {:target {:target-incremental-strategy {:type "append"}}})))))
+ (transform-with-target {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}}))))))
(deftest validate-merge-unique-key!-test
(testing "returns the key when every column is present in the target columns"
@@ -75,7 +83,7 @@
(testing "on OSS (no :database-routing premium feature) the check is a no-op"
(mt/with-premium-features #{}
(is (nil? (transforms-base.u/throw-if-db-routing-enabled!
- {:name "OSS transform"}
+ (assoc (transform-with-target {:type "table"}) :name "OSS transform")
(lib.metadata/database (mt/metadata-provider)))))))
(when config/ee-available?
(testing "with :database-routing premium feature enabled, the check throws on a routing-enabled database"
@@ -86,7 +94,7 @@
clojure.lang.ExceptionInfo
#".*database routing turned on"
(transforms-base.u/throw-if-db-routing-enabled!
- {:name "Routing transform"}
+ (assoc (transform-with-target {:type "table"}) :name "Routing transform")
(lib.metadata/database (mt/metadata-provider))))))))))
(deftest activate-table-syncs-despite-disable-auto-sync-test
@@ -107,80 +115,84 @@
(deftest ^:parallel full-incremental-run?-test
(testing "true for an incremental transform with no checkpoint yet"
(is (true? (transforms-base.u/full-incremental-run?
- {:target {:type "table-incremental"} :last_checkpoint_value nil}))))
+ (assoc (transform-with-target {:type "table-incremental"}) :last_checkpoint_value nil)))))
(testing "false for an incremental transform that has already recorded a watermark"
(is (false? (transforms-base.u/full-incremental-run?
- {:target {:type "table-incremental"} :last_checkpoint_value "42"}))))
+ (assoc (transform-with-target {:type "table-incremental"}) :last_checkpoint_value "42")))))
(testing "false for non-incremental targets regardless of checkpoint value"
(is (false? (transforms-base.u/full-incremental-run?
- {:target {:type "table"} :last_checkpoint_value nil})))
+ (assoc (transform-with-target {:type "table"}) :last_checkpoint_value nil))))
(is (false? (transforms-base.u/full-incremental-run?
- {:target {:type :table} :last_checkpoint_value nil}))))
+ (assoc (transform-with-target {:type :table}) :last_checkpoint_value nil)))))
(testing "accepts both string and keyword target types"
(is (true? (transforms-base.u/full-incremental-run?
- {:target {:type :table-incremental} :last_checkpoint_value nil})))))
+ (assoc (transform-with-target {:type :table-incremental}) :last_checkpoint_value nil))))))
+
+(defn- column
+ [m]
+ (merge {:lib/type :metadata/column :name "checkpoint"} m))
(deftest ^:parallel supported-checkpoint-column?-test
(testing "numeric and date/datetime columns are supported"
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Integer}))
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Float}))
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Date}))
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTime}))
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTimeWithTZ}))
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Instant})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Integer})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Float})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Date})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/DateTime})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/DateTimeWithTZ})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Instant}))))
(testing "coerced columns count via their effective type"
- (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/BigInteger
- :effective-type :type/Instant})))
+ (is (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/BigInteger
+ :effective-type :type/Instant}))))
(testing "time-only and non-orderable columns are not supported"
- (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Time})))
- (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/TimeWithTZ})))
- (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Text})))
- (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Boolean})))))
+ (is (not (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Time}))))
+ (is (not (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/TimeWithTZ}))))
+ (is (not (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Text}))))
+ (is (not (transforms-base.u/supported-checkpoint-column? (column {:base-type :type/Boolean}))))))
(deftest ^:parallel apply-lookback-test
(let [apply-lookback @#'transforms-base.u/apply-lookback]
(testing "temporal checkpoints are pushed back by value units"
(is (= (t/local-date-time 2026 1 27 21 0 4)
- (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4 :unit "day"})))
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) (column {:base-type :type/DateTime}) {:value 4 :unit "day"})))
(is (= (t/local-date-time 2026 1 31 19 0 4)
- (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 2 :unit "hour"})))
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) (column {:base-type :type/DateTime}) {:value 2 :unit "hour"})))
(is (= (t/local-date 2026 1 24)
- (apply-lookback (t/local-date 2026 1 31) {:base-type :type/Date} {:value 1 :unit "week"}))))
+ (apply-lookback (t/local-date 2026 1 31) (column {:base-type :type/Date}) {:value 1 :unit "week"}))))
(testing "a coerced numeric column (e.g. a unix timestamp) is temporal via its effective type"
(is (= (u.date/parse "2026-01-30T21:00:04Z")
(apply-lookback (u.date/parse "2026-01-31T21:00:04Z")
- {:base-type :type/BigInteger :effective-type :type/Instant}
+ (column {:base-type :type/BigInteger :effective-type :type/Instant})
{:value 1 :unit "day"}))))
(testing "a lookback on a non-temporal checkpoint throws a user-facing error"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"only supported for date or datetime"
- (apply-lookback (biginteger 42) {:base-type :type/Integer} {:value 4 :unit "day"}))))
+ (apply-lookback (biginteger 42) (column {:base-type :type/Integer}) {:value 4 :unit "day"}))))
(testing "a lookback on a time-only checkpoint throws a user-facing error"
;; time-only watermarks wrap at midnight, and day-based units would throw inside u.date/add
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"only supported for date or datetime"
- (apply-lookback (t/local-time 21 0 4) {:base-type :type/Time} {:value 2 :unit "hour"})))
+ (apply-lookback (t/local-time 21 0 4) (column {:base-type :type/Time}) {:value 2 :unit "hour"})))
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"only supported for date or datetime"
- (apply-lookback (t/offset-time 21 0 4) {:base-type :type/TimeWithTZ} {:value 1 :unit "day"}))))
+ (apply-lookback (t/offset-time 21 0 4) (column {:base-type :type/TimeWithTZ}) {:value 1 :unit "day"}))))
(testing "a lookback without a unit throws a user-facing error"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"requires a unit"
- (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4}))))
+ #"Invalid input"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) (column {:base-type :type/DateTime}) {:value 4}))))
(testing "a lookback with a non-positive value throws instead of silently skipping data"
;; a negative window would move `lo` forward past the watermark, permanently skipping rows
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"positive integer value"
- (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 0 :unit "day"})))
+ #"Invalid input"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) (column {:base-type :type/DateTime}) {:value 0 :unit "day"})))
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"positive integer value"
- (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value -4 :unit "day"}))))))
+ #"Invalid input"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) (column {:base-type :type/DateTime}) {:value -4 :unit "day"}))))))
(deftest ^:parallel checkpoint-compare-test
(let [checkpoint-compare @#'transforms-base.u/checkpoint-compare]
@@ -200,18 +212,21 @@
(is (= {} (transforms-base.u/checkpoint-span-attrs nil))))
(testing "field-id only (no lo/hi) yields just :transform/checkpoint-field-id"
(is (= {:transform/checkpoint-field-id 42}
- (transforms-base.u/checkpoint-span-attrs {:checkpoint-filter-field-id 42}))))
+ (transforms-base.u/checkpoint-span-attrs {:column (column {:base-type :type/Integer})
+ :checkpoint-filter-field-id 42}))))
(testing "numeric lo/hi are encoded as strings"
(is (= {:transform/checkpoint-field-id 7
:transform/checkpoint-lo "10"
:transform/checkpoint-hi "100"}
(transforms-base.u/checkpoint-span-attrs
- {:checkpoint-filter-field-id 7
+ {:column (column {:base-type :type/Integer})
+ :checkpoint-filter-field-id 7
:lo {:value 10}
:hi {:value 100}}))))
(testing "temporal hi is formatted as an ISO string"
(let [attrs (transforms-base.u/checkpoint-span-attrs
- {:checkpoint-filter-field-id 9
+ {:column (column {:base-type :type/DateTime})
+ :checkpoint-filter-field-id 9
:hi {:value (t/local-date-time 2024 1 16 10 0 0)}})]
(is (= 9 (:transform/checkpoint-field-id attrs)))
(is (string? (:transform/checkpoint-hi attrs)))
diff --git a/test/metabase/transforms_rest/api/transform_runs_test.clj b/test/metabase/transforms_rest/api/transform_runs_test.clj
index 9f8c0b957ed..75ffbe8902c 100644
--- a/test/metabase/transforms_rest/api/transform_runs_test.clj
+++ b/test/metabase/transforms_rest/api/transform_runs_test.clj
@@ -3,6 +3,8 @@
DAG-reprocess runs, and standalone transform runs) — and GET /api/transform/run/:run-id."
(:require
[clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.transforms.test-util :refer [parse-instant
@@ -255,10 +257,8 @@
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-temp [:model/Database {db-id :id} {}
:model/Transform {transform-id :id} {:name "Hidden Transform"
- :source {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"}}}}
+ :source {:type :query
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "SELECT 1")}}
:model/TransformRun {run-id :id} {:transform_id transform-id
:transform_name "Hidden Transform"}]
;; deleting the source database makes the transform unreadable to data analysts
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index b4fb4068435..55d37a80703 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1,13 +1,11 @@
(ns ^:mb/driver-tests metabase.transforms-rest.api.transform-test
"Tests for /api/transform endpoints."
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.transforms-rest.api.transform-test]}}}}}}
(:require
[clojure.test :refer :all]
[medley.core :as m]
[metabase.driver :as driver]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.permissions.core :as perms]
@@ -59,6 +57,11 @@
(map :id))
(search.tu/search-results search-term)))
+(defn- table-query
+ [table-id]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp table-id))))
+
(defn- make-query
"Create a query filtering products by category, using shared utility.
Returns a legacy MBQL query structure for API compatibility."
@@ -93,7 +96,7 @@
schema (get-test-schema)
response (mt/user-http-request user :post 200 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -125,7 +128,7 @@
schema (get-test-schema)
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Update Schema Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -154,7 +157,7 @@
(let [query (make-query "Gadget")
request (fn [schema]
{:name "No Schema Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -179,7 +182,7 @@
schema (get-test-schema)
response (mt/user-http-request user :post 200 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -203,7 +206,7 @@
schema (get-test-schema)
response (mt/user-http-request user :post 400 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -225,7 +228,7 @@
schema (get-test-schema)
response (mt/user-http-request user :post 400 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -246,7 +249,7 @@
rasta-id (mt/user->id :rasta)
response (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform with explicit owner"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -269,7 +272,7 @@
schema (get-test-schema)
response (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform with external owner"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -291,7 +294,7 @@
schema (get-test-schema)
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform for owner update"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -334,7 +337,7 @@
schema (get-test-schema)
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform for owner update"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -344,7 +347,7 @@
(testing "Update query to a query with an optional param"
(mt/user-http-request :crowberto :put 200
(format "transform/%s" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query new-query}}))))))))
(deftest update-transform-query-with-required-param-test
@@ -357,7 +360,7 @@
schema (get-test-schema)
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform for owner update"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -370,7 +373,7 @@
{"id" (assoc tag :required true)})
response (mt/user-http-request :crowberto :put 400
(format "transform/%s" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query new-query}})]
(is (= "You'll need to pick a value for 'ID' before this query can run."
(:message response))))))))))
@@ -385,7 +388,7 @@
schema (get-test-schema)
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Transform for owner update"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -394,7 +397,7 @@
new-query (lib/native-query (mt/metadata-provider) "select * from foo where {{id}} = id")
response (mt/user-http-request :crowberto :put 400
(format "transform/%s" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query new-query}})]
(is (= "Cannot run the query: missing required parameters: #{\"id\"}"
(:message response))))))))))
@@ -408,11 +411,11 @@
(with-transform-db-perms!
(testing "MBQL query transforms are detected as :mbql"
(with-transform-cleanup! [table-name "mbql_transform"]
- (let [mbql-query (mt/mbql-query transforms_products)
+ (let [mbql-query (table-query (mt/id :transforms_products))
schema (get-test-schema)
response (mt/user-http-request user :post 200 "transform"
{:name "MBQL Transform"
- :source {:type "query"
+ :source {:type :query
:query mbql-query}
:target {:type "table"
:schema schema
@@ -423,7 +426,7 @@
(let [schema (get-test-schema)
response (mt/user-http-request user :post 200 "transform"
{:name "Native Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:schema schema
@@ -439,11 +442,11 @@
(with-transform-db-perms!
(with-transform-cleanup! [table-name "type_update_transform"]
(let [native-query (lib/native-query (mt/metadata-provider) "SELECT 1")
- mbql-query (mt/mbql-query transforms_products)
+ mbql-query (table-query (mt/id :transforms_products))
schema (get-test-schema)
created (mt/user-http-request user :post 200 "transform"
{:name "Native Transform"
- :source {:type "query"
+ :source {:type :query
:query native-query}
:target {:type "table"
:schema schema
@@ -452,7 +455,7 @@
(testing "Type automatically changes to mbql when updating to an MBQL query"
(let [updated (mt/user-http-request user :put 200
(format "transform/%s" (:id created))
- {:source {:type "query"
+ {:source {:type :query
:query mbql-query}})]
(is (= "mbql" (:source_type updated))))))))))))))
@@ -469,7 +472,7 @@
schema (get-test-schema)
response (mt/user-http-request user :post 200 "transform"
{:name "Test Transform"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -488,7 +491,7 @@
(let [query (make-query "Gadget")
schema (get-test-schema)
transform-payload {:name "Original Transform"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -511,7 +514,7 @@
(let [query (make-query "Gadget")
schema (get-test-schema)
transform-payload {:name "Test Run Transform"
- :source {:type "query"
+ :source {:type :query
:query query}
:target {:type "table"
:schema schema
@@ -533,7 +536,7 @@
(let [table-name (str "disabled_transform_" (u/generate-nano-id))
response (mt/user-http-request :crowberto :post 402 "transform"
{:name "Disabled Transform"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -556,7 +559,7 @@
(mt/with-data-analyst-role! (mt/user->id :lucky)
(with-transform-cleanup! [table-name "test_read_run"]
(mt/with-temp [:model/Transform transform {:name "Test Read Run Transform"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -582,7 +585,7 @@
(with-transform-cleanup! [table-name "gadget_products"]
(let [body {:name "Gadget Products"
:description "Desc"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -676,7 +679,7 @@
(with-transform-db-perms!
(with-transform-cleanup! [table-name "reqidx_products"]
(let [body {:name "Reqidx Products"
- :source {:type "query" :query (make-query "Gadget")}
+ :source {:type :query :query (make-query "Gadget")}
:target {:type "table" :schema (get-test-schema) :name table-name}}
created (mt/user-http-request user :post 200 "transform" body)
get-resp (mt/user-http-request user :get 200 (format "transform/%s" (:id created)))
@@ -701,7 +704,7 @@
(with-transform-cleanup! [table-name "test_readable"]
(let [body {:name "Test Transform"
:description "Test"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -727,7 +730,7 @@
"User with transforms permission should be able to read the source database")))))))))))))))
(defn- ->transform [transform-name query]
- {:source {:type "query",
+ {:source {:type :query,
:query query}
:name transform-name
:target {:schema "public"
@@ -740,16 +743,14 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-temp [:model/Table {table :id} {:schema "public", :name "orders_2"}
:model/Field _ {:table_id table, :name "foo"}
- :model/Transform parent (-> (->transform "transform1" (mt/mbql-query orders))
+ :model/Transform parent (-> (->transform "transform1" (table-query (mt/id :orders)))
(assoc :target_table_id table))
- :model/Transform child (-> (->transform "transform2" (mt/mbql-query nil {:source-table table}))
+ :model/Transform child (-> (->transform "transform2" (table-query table))
(assoc-in [:target :name] "orders_3"))]
(let [deps-resp (mt/user-http-request user :get 200 (format "transform/%s/dependencies" (:id child)))]
(is (=? [{:name "transform1"
:source {:type "query"
- :query {:database (mt/id)
- :lib/type "mbql/query"
- :stages [{:source-table (mt/id :orders)}]}}
+ :query (mt/obj->json->obj (table-query (mt/id :orders)))}
:id (:id parent)
:entity_id (:entity_id parent)
:target {:schema "public", :name "orders_2", :type "table"}}]
@@ -767,7 +768,7 @@
(let [query2 (make-query "None")
resp (mt/user-http-request user :post 200 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -798,7 +799,7 @@
table2-name "doohickey_products"]
(let [query2 (make-query "Doohickey")
original {:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -827,7 +828,7 @@
(with-transform-cleanup! [table-name "gadget_products"]
(let [resp (mt/user-http-request user :post 200 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -844,7 +845,7 @@
(with-transform-cleanup! [table-name "gadget_products"]
(let [resp (mt/user-http-request user :post 200 "transform"
{:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -924,7 +925,7 @@
:name "doohickey_products"}]
(let [query2 (make-query "Doohickey")
original {:name "Gadget Products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target target1}
{transform-id :id} (mt/user-http-request user :post 200 "transform"
@@ -1149,35 +1150,23 @@
(mt/with-premium-features #{}
(testing "GET /api/transform/run - filter by multiple tags (union)"
(mt/with-temp [:model/Transform transform1 {:name "Transform with tag1"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name (str "test_table_1_" (u/generate-nano-id))}}
:model/Transform transform2 {:name "Transform with both tags"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 2"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 2")}
:target {:type "table"
:name (str "test_table_2_" (u/generate-nano-id))}}
:model/Transform transform3 {:name "Transform with tag2"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 3"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 3")}
:target {:type "table"
:name (str "test_table_3_" (u/generate-nano-id))}}
:model/Transform transform4 {:name "Untagged Transform"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 4"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 4")}
:target {:type "table"
:name (str "test_table_4_" (u/generate-nano-id))}}
:model/TransformTag tag1 {}
@@ -1203,19 +1192,13 @@
(mt/with-premium-features #{}
(testing "GET /api/transform/run - combine transform ID and status filters"
(mt/with-temp [:model/Transform transform1 {:name "Transform 1"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name (str "test_table_1_" (u/generate-nano-id))}}
:model/Transform transform2 {:name "Transform 2"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 2"
- :template-tags {}}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 2")}
:target {:type "table"
:name (str "test_table_2_" (u/generate-nano-id))}}
:model/TransformRun _run1 {:transform_id (:id transform1) :status "succeeded"}
@@ -1306,7 +1289,7 @@
transform-id))
gadget-req {:name "Gadget Products"
:description "The gadget products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -1314,7 +1297,7 @@
transform-id (test-transform-revisions :post "transform" gadget-req 1)
widget-req {:name "Widget Products"
:description "The widget products"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Widget")}
:tag_ids [4]
:target {:type "table"
@@ -1331,10 +1314,8 @@
(mt/user-http-request :rasta :get 403 (str "transform/" (:id transform)))
(mt/user-http-request :rasta :post 403 "transform"
{:name "Test"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table" :name (mt/random-name)}})
(mt/user-http-request :rasta :put 403 (str "transform/" (:id transform))
{:name "Updated"})
@@ -1361,7 +1342,7 @@
;; Create transform owned by rasta, but run by crowberto
transform (mt/user-http-request :crowberto :post 200 "transform"
{:name "Attribution Test"
- :source {:type "query"
+ :source {:type :query
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
@@ -1472,10 +1453,8 @@
(with-transform-db-perms!
(with-transforms-api-users! [user]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name (mt/random-name)}}
:model/TransformTag tag1 {:name "update-tag-1"}
@@ -1501,10 +1480,8 @@
(with-transform-db-perms!
(with-transforms-api-users! [user]
(mt/with-temp [:model/Transform transform {:name "Transform With Tags"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name "tagged_table"}}
:model/TransformTag tag1 {:name "get-tag-1"}
@@ -1516,10 +1493,8 @@
(is (= [(:id tag1) (:id tag2)] (sort (:tag_ids fetched))))))
(testing "Transform without tags returns empty array"
(mt/with-temp [:model/Transform transform2 {:name "Transform Without Tags"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name "untagged_table"}}]
(let [fetched (mt/user-http-request user :get 200 (str "transform/" (:id transform2)))]
@@ -1531,17 +1506,13 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(with-transforms-api-users! [user]
(mt/with-temp [:model/Transform transform1 {:name "Transform 1"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name "list_table_1"}}
:model/Transform transform2 {:name "Transform 2"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 2"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 2")}
:target {:type "table"
:name "list_table_2"}}
:model/TransformTag tag1 {:name "list-tag-1"}
@@ -1563,10 +1534,8 @@
(with-transform-db-perms!
(with-transforms-api-users! [user]
(mt/with-temp [:model/Transform transform {:name "Transform for Delete Test"
- :source {:type "query"
- :query {:database (mt/id)
- :type "native"
- :native {:query "SELECT 1"}}}
+ :source {:type :query
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:name "delete_test_table"}}
:model/TransformTag tag2 {:name "tag-to-keep"}]
@@ -1713,7 +1682,7 @@
(testing "Rejects unsupported checkpoint column type (text)"
(let [response (mt/user-http-request :crowberto :post 400 "transform"
{:name "Invalid Incremental Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category, price FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1736,7 +1705,7 @@
(let [;; Create a non-incremental transform first
created (mt/user-http-request :crowberto :post 200 "transform"
{:name "Test Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category FROM transforms_products")}
:target {:type "table"
@@ -1745,7 +1714,7 @@
(testing "Rejects update to unsupported checkpoint column type (text)"
(let [response (mt/user-http-request :crowberto :put 400
(format "transform/%d" (:id created))
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1772,7 +1741,7 @@
(mt/with-temp [:model/TransformTag {tag-id :id} {:name "stale-checkpoint-tag"}
:model/Transform {transform-id :id}
{:name "Stale Checkpoint Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1789,17 +1758,15 @@
(is (= [] (:tag_ids updated)))))))))))))
(defn- native-query-with-table-tag
- "Build a native query map with a `source_table` table template tag referring to `table-id`."
+ "Build a native query with a `source_table` table template tag referring to `table-id`."
[sql table-id]
- {:database (mt/id)
- :type :native
- :native {:query sql
- :template-tags {"source_table" {:id "source_table"
- :name "source_table"
- :display-name "Source Table"
- :type "table"
- :table-id table-id
- :required true}}}})
+ (lib/with-template-tags (lib/native-query (mt/metadata-provider) sql)
+ {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type :table
+ :table-id table-id
+ :required true}}))
(deftest incremental-table-tag-validated-on-create-test
(mt/with-premium-features #{:advanced-permissions :transforms-basic :hosting}
@@ -1811,7 +1778,7 @@
(testing "with a checkpoint field selected"
(let [response (mt/user-http-request :crowberto :post 400 "transform"
{:name "No Table Tag"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1825,7 +1792,7 @@
(testing "even without a checkpoint field selected"
(let [response (mt/user-http-request :crowberto :post 400 "transform"
{:name "No Table Tag No Checkpoint"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name FROM transforms_products")}
:target {:type "table-incremental"
@@ -1846,7 +1813,7 @@
(with-transform-cleanup! [table-name "remove_table_tag_test"]
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Has Table Tag"
- :source {:type "query"
+ :source {:type :query
:query (native-query-with-table-tag
"SELECT id, name FROM {{source_table}}"
(mt/id :transforms_products))
@@ -1858,7 +1825,7 @@
:target-incremental-strategy {:type "append"}}}]
(let [response (mt/user-http-request :crowberto :put 400
(format "transform/%d" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1869,13 +1836,13 @@
(with-transform-cleanup! [table-name "switch_no_tag_test"]
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Plain Transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name FROM transforms_products")}
:target {:type "table" :schema schema :name table-name}}]
(let [response (mt/user-http-request :crowberto :put 400
(format "transform/%d" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
@@ -1900,7 +1867,7 @@
num-field-id (mt/id :transforms_products :price)
payload (fn [table-name checkpoint-field-id lookback target-strategy]
{:name "Lookback Transform"
- :source {:type "query"
+ :source {:type :query
:query query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id checkpoint-field-id
@@ -1958,7 +1925,7 @@
(with-transform-cleanup! [table-name "lookback_update"]
(mt/with-temp [:model/Transform {transform-id :id}
{:name "Append Incremental"
- :source {:type "query"
+ :source {:type :query
:query query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id}}
@@ -1969,7 +1936,7 @@
(testing "adding a lookback to an existing incremental transform is accepted"
(let [updated (mt/user-http-request :crowberto :put 200
(format "transform/%d" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id
@@ -1979,7 +1946,7 @@
(testing "switching the checkpoint to a numeric column while a lookback is set is rejected"
(let [response (mt/user-http-request :crowberto :put 400
(format "transform/%d" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id num-field-id
@@ -1989,7 +1956,7 @@
(testing "adding a lookback together with a switch to the merge strategy is accepted"
(let [updated (mt/user-http-request :crowberto :put 200
(format "transform/%d" transform-id)
- {:source {:type "query"
+ {:source {:type :query
:query query
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id ts-field-id
@@ -2016,9 +1983,9 @@
(let [search-term (str "transform-search-" (u/generate-nano-id))
query-name (str search-term "_query")
python-name (str search-term "_python")
- query-source {:type "query"
+ query-source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
- python-source {:type "python"
+ python-source {:type :python
:body "print('hello world')"
:source-tables []
:source-database (mt/id)}]
@@ -2046,7 +2013,7 @@
(search.tu/with-temp-index-table-if-supported
(let [search-term (str "transform-search-" (u/generate-nano-id))
query-name (str search-term "_query")
- query-source {:type "query"
+ query-source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}]
(mt/with-temp [:model/Transform {transform-id :id} {:name query-name
:source query-source
@@ -2068,10 +2035,10 @@
(let [search-term (str "transform-search-" (u/generate-nano-id))
native-name (str search-term "_native")
mbql-name (str search-term "_mbql")
- native-source {:type "query"
+ native-source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
- mbql-source {:type "query"
- :query (mt/mbql-query transforms_products)}]
+ mbql-source {:type :query
+ :query (table-query (mt/id :transforms_products))}]
(mt/with-temp [:model/Transform {native-id :id} {:name native-name
:source native-source
:target {:type "table"
@@ -2156,7 +2123,7 @@
:required true :default ["Gadget"]}})
{id :id} (mt/user-http-request user :post 200 "transform"
{:name "Field Filter Transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:type "table"
:schema (get-test-schema)
:name table-name}})]
@@ -2178,7 +2145,7 @@
:table-id source-table-id}})
{id :id} (mt/user-http-request user :post 200 "transform"
{:name "Table Tag Transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:type "table"
:schema (get-test-schema)
:name table-name}})]
@@ -2200,7 +2167,7 @@
(testing "POST /api/transform"
(mt/user-http-request user :post 409 "transform"
{:name "Colliding Transform"
- :source {:type "query" :query (make-query "Gadget")}
+ :source {:type :query :query (make-query "Gadget")}
:target {:type "table"
:schema schema
:name existing-table-name}}))
@@ -2208,7 +2175,7 @@
(with-transform-cleanup! [table-name "collision_source"]
(let [created (mt/user-http-request user :post 200 "transform"
{:name "Non-colliding Transform"
- :source {:type "query" :query (make-query "Gadget")}
+ :source {:type :query :query (make-query "Gadget")}
:target {:type "table"
:schema schema
:name table-name}})]
@@ -2227,7 +2194,7 @@
(with-transform-cleanup! [table-name "gadget_products_recreate"]
(let [target {:type "table" :schema (get-test-schema) :name table-name}
definition {:name "Recreate Target Transform"
- :source {:type "query" :query (make-query "Gadget")}
+ :source {:type :query :query (make-query "Gadget")}
:target target}
{id :id} (mt/user-http-request user :post 200 "transform" definition)]
(test-run! user id)
@@ -2237,18 +2204,6 @@
(test-run! user id)
(is (true? (transforms-base.u/target-table-exists? definition))))))))))))
-(deftest get-transform-with-deleted-database-test
- (testing "GET /api/transform/:id degrades gracefully when its source database has been deleted"
- (mt/with-premium-features #{:advanced-permissions :transforms-basic :hosting}
- (mt/with-temp [:model/Database db {}]
- (let [mp (lib.metadata.jvm/application-database-metadata-provider (:id db))
- query (lib/native-query mp "select 1")]
- (mt/with-temp [:model/Transform transform {:source {:type "query" :query query}
- :target {:type "table" :name (mt/random-name)}}]
- (t2/delete! :model/Database :id (:id db))
- (is (=? {:id (:id transform)}
- (mt/user-http-request :crowberto :get 200 (str "transform/" (:id transform)))))))))))
-
(deftest cancel-transform-run-endpoint-test
(testing "POST /api/transform/:id/cancel"
(mt/with-premium-features #{:advanced-permissions :transforms-basic :hosting}
@@ -2259,7 +2214,7 @@
"SELECT a FROM (SELECT pg_sleep(5)) x, generate_series(1, 5) a")
{id :id} (mt/user-http-request :crowberto :post 200 "transform"
{:name "Slow Transform"
- :source {:type "query" :query query}
+ :source {:type :query :query query}
:target {:type "table"
:schema (get-test-schema)
:name table-name}})]
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index 4545be201e5..2070aaf49e2 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -430,9 +430,9 @@
(deftest query-metadata-transform-hydration-test
(testing "GET /api/table/:id/query_metadata hydrates the source :transform based on transforms availability (GDGT-2523)"
(mt/with-temp [:model/Transform transform {:name "My transform"
- :source {:type "query"
+ :source {:type :query
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
- :target {:database (mt/id) :table "my_transform_table"}}
+ :target {:type "table" :database (mt/id) :name "my_transform_table"}}
:model/Table {table-id :id} {:db_id (mt/id)
:transform_id (:id transform)}]
(testing "self-hosted (not hosted) without :transforms-basic still hydrates the source transform"
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index d370e25eede..86baca5880a 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -15,6 +15,7 @@
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.id :as lib.schema.id]
@@ -296,18 +297,14 @@
:model/Segment _ {:table_id table-id-2}
:model/Transform _ {:name "Source DB transform"
:source {:type :query
- :query {:database db-id
- :type :native
- :native {:query "select 1"}}}
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "select 1")}
:target {:type "table"
:database db-id
:schema "PUBLIC"
:name "source_db_transform_target"}}
:model/Transform _ {:name "Target DB transform"
:source {:type :query
- :query {:database db-id
- :type :native
- :native {:query "select 1"}}}
+ :query (lib/native-query (lib-be/application-database-metadata-provider db-id) "select 1")}
:target {:type "table"
:database db-id
:schema "PUBLIC"