Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Incremental transforms (#65842)

Stas Gavrylova07c3b654e5d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 0d70159f6ec..5593084cc0d 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -680,6 +680,7 @@
metabase.lib.join.util
metabase.lib.metadata
metabase.lib.metadata.cached-provider
+ metabase.lib.metadata.column
metabase.lib.metadata.invocation-tracker
metabase.lib.metadata.protocols
metabase.lib.metadata.result-metadata
@@ -2276,6 +2277,8 @@
app-db
config
driver
+ lib
+ lib-be
query-processor
settings
util
diff --git a/deps.edn b/deps.edn
index e0f6f6aa4d8..f8330dffce1 100644
--- a/deps.edn
+++ b/deps.edn
@@ -860,6 +860,13 @@
{:exec-args {:only ["test/metabase/notification"
"test/metabase/channel"]}}
+ ;; test the transforms feature
+ ;; clj -X:dev:ee:ee-dev:test:test/transforms
+ :test/transforms
+ {:exec-args {:only ["enterprise/backend/test/metabase_enterprise/transforms"
+ "enterprise/backend/test/metabase_enterprise/transforms_python"]}}
+
+
;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP
;;
;; clj -X:dev:ee:ee-dev:test:test/mbql
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index f80d8ba945e..bd866e413c5 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -8,6 +8,8 @@ title: Driver interface changelog
- Added a `:collate` feature for drivers that support collation settings on text fields
+- Added `metabase.driver/compile-insert` to implement incremental transforms.
+
## Metabase 0.57.0
- `driver/field-reference-mlv2` is now deprecated, and is no longer used. Please remove your implementations.
diff --git a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
index faf742701b5..84dd0a0a34f 100644
--- a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
@@ -420,10 +420,10 @@ LIMIT
cy.findByLabelText("Table name").type(SOURCE_TABLE);
cy.button("Save").click();
cy.wait("@createTransform");
- cy.findByText("A table with that name already exists.").should(
- "be.visible",
- );
});
+ H.undoToast()
+ .findByText("A table with that name already exists.")
+ .should("be.visible");
});
it("should be able to create a new schema when saving a transform", () => {
@@ -1012,14 +1012,14 @@ LIMIT
createMbqlTransform({ visitTransform: true });
H.DataStudio.Transforms.targetTab().click();
getTransformsContent()
- .findByText("Edit this table’s metadata")
+ .findByText("Edit this table's metadata")
.should("not.exist");
cy.log("after table creation");
H.DataStudio.Transforms.runTab().click();
runTransformAndWaitForSuccess();
H.DataStudio.Transforms.targetTab().click();
- getTransformsContent().findByText("Edit this table’s metadata").click();
+ getTransformsContent().findByText("Edit this table's metadata").click();
H.DataModel.TableSection.clickField("Name");
H.DataModel.FieldSection.getNameInput().clear().type("New name").blur();
cy.wait("@updateField");
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index 7c49e9e4912..11f8dbddf54 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.transforms.api
(:require
+ [macaw.core :as macaw]
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
[metabase-enterprise.transforms.canceling :as transforms.canceling]
@@ -8,47 +9,39 @@
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation]
[metabase-enterprise.transforms.ordering :as transforms.ordering]
+ [metabase-enterprise.transforms.schema :as transforms.schema]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.api.util.handlers :as handlers]
+ [metabase.driver :as driver]
+ [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
- [metabase.queries.schema :as queries.schema]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.schema :as qp.schema]
[metabase.request.core :as request]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.jvm :as u.jvm]
+ [metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[ring.util.response :as response]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.sql PreparedStatement)
+ (net.sf.jsqlparser.statement.select PlainSelect)))
(comment metabase-enterprise.transforms.api.transform-job/keep-me
metabase-enterprise.transforms.api.transform-tag/keep-me)
(set! *warn-on-reflection* true)
-(mr/def ::transform-source
- [:multi {:dispatch (comp keyword :type)}
- [:query
- [:map
- [:type [:= "query"]]
- [:query ::queries.schema/query]]]
- [:python
- [:map {:closed true}
- [:source-database {:optional true} :int]
- [:source-tables [:map-of :string :int]]
- [:type [:= "python"]]
- [:body :string]]]])
-
-(mr/def ::transform-target
- [:map
- [:database {:optional true} :int]
- [:type [:enum "table"]]
- [:schema {:optional true} [:or ms/NonBlankString :nil]]
- [:name :string]])
+(mr/def ::transform-source ::transforms.schema/transform-source)
+
+(mr/def ::transform-target ::transforms.schema/transform-target)
(mr/def ::run-trigger
[:enum "none" "global-schedule"])
@@ -70,13 +63,13 @@
[transform]
(let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))
(deferred-tru "The target database cannot be found."))
- feature (transforms.util/required-database-feature transform)]
+ features (transforms.util/required-database-features transform)]
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
(api/check-400 (not (:is_audit database))
(deferred-tru "Cannot run transforms on audit databases."))
- (api/check-400 (driver.u/supports? (:engine database) feature database)
- (deferred-tru "The database does not support the requested transform target type."))
+ (api/check-400 (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
+ (deferred-tru "The database does not support the requested transform features."))
(api/check-400 (not (transforms.util/db-routing-enabled? database))
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
@@ -117,13 +110,14 @@
body :- [:map
[:name :string]
[:description {:optional true} [:maybe :string]]
- [:source ::transform-source]
- [:target ::transform-target]
+ [:source ::transforms.schema/transform-source]
+ [:target ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check-superuser)
(check-database-feature body)
(check-feature-enabled! body)
+
(api/check (not (transforms.util/target-table-exists? body))
403
(deferred-tru "A table with that name already exists."))
@@ -200,8 +194,8 @@
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
- [:source {:optional true} ::transform-source]
- [:target {:optional true} ::transform-target]
+ [:source {:optional true} ::transforms.schema/transform-source]
+ [:target {:optional true} ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check-superuser)
@@ -281,6 +275,86 @@
:run_id run-id})
(assoc :status 202)))))
+(defn- extract-columns-from-query
+ "Attempts to extract column names from an MBQL query without executing it.
+
+ Returns a vector of column names (as strings), or nil if extraction fails.
+
+ The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()
+ to inspect the query structure. This works for most modern JDBC drivers but may not
+ be supported by all drivers or for all query types."
+ [driver database-id query]
+ (try
+ (let [{:keys [query]} (qp.compile/compile query)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database-id
+ {}
+ (fn [conn]
+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
+ (when-let [rsmeta (.getMetaData stmt)]
+ (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]
+ (seq (mapv :name columns))))))))
+ (catch Exception e
+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
+ nil)))
+
+(defn- simple-native-query?
+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion."
+ [sql-string]
+ (try
+ (let [^PlainSelect parsed (macaw/parsed-query sql-string)]
+ (cond
+ (not (instance? PlainSelect parsed))
+ {:is_simple false
+ :reason "Not a simple SELECT"}
+
+ (.getLimit parsed)
+ {:is_simple false
+ :reason "Contains a LIMIT"}
+
+ (.getOffset ^PlainSelect parsed)
+ {:is_simple false
+ :reason "Contains an OFFSET"}
+
+ (seq (.getWithItemsList ^PlainSelect parsed))
+ {:is_simple false
+ :reason "Contains a CTE"}
+
+ :else
+ {:is_simple true}))
+ (catch Exception e
+ (log/debugf e "Failed to parse query: %s" (ex-message e))
+ {:is_simple false})))
+
+(api.macros/defendpoint :post "/is-simple-query"
+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion"
+ [_route-params
+ _query-params
+ {:keys [query]} :- [:map [:query string?]]]
+ (api/check-superuser)
+ (simple-native-query? query))
+
+(api.macros/defendpoint :post "/extract-columns"
+ "Extract column names from an MBQL query without executing it.
+
+ Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],
+ which handles parameterized queries with template tags. Then extracts column names
+ using PreparedStatement metadata.
+
+ Returns a map with a :columns key containing a vector of column names (strings).
+ If extraction fails, returns nil for :columns."
+ [_route-params
+ _query-params
+ {:keys [query]} :- [:map
+ [:query ::qp.schema/any-query]]]
+ (api/check-superuser)
+ (let [database-id (:database query)
+ database (api/check-404 (t2/select-one :model/Database :id database-id))
+ driver-name (driver/the-initialized-driver (:engine database))
+ columns (extract-columns-from-query driver-name database-id query)]
+ {:columns columns}))
+
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/transform` routes."
(handlers/routes
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index 7f4206eee54..e05ac25b569 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -6,6 +6,7 @@
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
+ [metabase.query-processor.compile :as qp.compile]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
@@ -14,15 +15,22 @@
(mr/def ::transform-details
[:map
- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
[:conn-spec :any]
- [:query :string]
+ [:query ::qp.compile/compiled]
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
(mr/def ::transform-opts
[:map
[:overwrite? :boolean]])
+(defn- transform-opts [{:keys [transform-type]}]
+ (case transform-type
+ :table {:overwrite? true}
+
+ ;; once we have more than just append, dispatch on :target-incremental-strategy
+ :table-incremental {}))
+
(defn run-mbql-transform!
"Run `transform` and sync its target table.
@@ -34,21 +42,23 @@
(try
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
- feature (transforms.util/required-database-feature transform)
transform-details {:db-id db
+ :database database
:transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
- :query (transforms.util/compile-source source)
+ :query (transforms.util/compile-source transform)
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
- opts {:overwrite? true}]
+ opts (transform-opts transform-details)
+ features (transforms.util/required-database-features transform)]
+
(when (transforms.util/db-routing-enabled? database)
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
{:driver driver, :database database})))
- (when-not (driver.u/supports? driver feature database)
+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
- {:driver driver, :database database, :feature feature})))
+ {:driver driver, :database database, :features features})))
;; mark the execution as started and notify any observers
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
index 19391a770bc..5088b912630 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
@@ -34,7 +34,8 @@
mi/json-out-without-keywordization
(update-keys keyword)
(m/update-existing :query lib-be/normalize-query)
- (m/update-existing :type keyword)))
+ (m/update-existing :type keyword)
+ (m/update-existing :source-incremental-strategy #(update-keys % keyword))))
(defn- transform-source-in [m]
(-> m
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
index cc18d98e565..900256f3194 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
@@ -18,8 +18,8 @@
(set! *warn-on-reflection* true)
(defmethod transforms.i/table-dependencies :query
- [transform]
- (let [query (-> (get-in transform [:source :query])
+ [{:keys [source]}]
+ (let [query (-> (:query source)
transforms.util/massage-sql-query
qp.preprocess/preprocess)
driver (-> query
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj
new file mode 100644
index 00000000000..272d6485cc1
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj
@@ -0,0 +1,62 @@
+(ns metabase-enterprise.transforms.schema
+ (:require
+ [metabase.lib.metadata.column :as lib.metadata.column]
+ [metabase.queries.schema :as queries.schema]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
+
+(mr/def ::checkpoint-strategy
+ [:map
+ [:type [:= "checkpoint"]]
+ ;; for native
+ [:checkpoint-filter {:optional true} :string]
+ ;; for mbql and python
+ [:checkpoint-filter-unique-key {:optional true}
+ ::lib.metadata.column/column-unique-key]])
+
+(mr/def ::source-incremental-strategy
+ [:multi {:dispatch :type}
+ ["checkpoint" ::checkpoint-strategy]])
+
+(mr/def ::transform-source
+ [:multi {:dispatch (comp keyword :type)}
+ [:query
+ [:map
+ [:type [:= "query"]]
+ [:query ::queries.schema/query]
+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]
+ [:python
+ [:map
+ [:source-database {:optional true} :int]
+ ;; NB: if source is checkpoint, only one table allowed
+ [:source-tables [:map-of :string :int]]
+ [:type [:= "python"]]
+ [:body :string]
+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])
+
+(mr/def ::append-config
+ [:map [:type [:= "append"]]])
+
+(mr/def ::target-incremental-strategy
+ [:multi {:dispatch :type}
+ ["append" ::append-config]])
+
+(mr/def ::table-target
+ [:map
+ [:database {:optional true} :int]
+ [:type [:= "table"]]
+ [:schema {:optional true} [:or ms/NonBlankString :nil]]
+ [:name :string]])
+
+(mr/def ::table-incremental-target
+ [:map
+ [:database {:optional true} :int]
+ [:type [:= "table-incremental"]]
+ [:schema {:optional true} [:or ms/NonBlankString :nil]]
+ [:name :string]
+ [:target-incremental-strategy ::target-incremental-strategy]])
+
+(mr/def ::transform-target
+ [:multi {:dispatch :type}
+ ["table" ::table-target]
+ ["table-incremental" ::table-incremental-target]])
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 2e17951c7c8..f4f57043132 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -8,16 +8,21 @@
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.settings :as transforms.settings]
[metabase.driver :as driver]
+ [metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.query :as lib.query]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
+ [metabase.query-processor :as qp]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.sync.core :as sync]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
+ [metabase.util.honey-sql-2 :as h2x]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
@@ -124,7 +129,7 @@
(finally
(canceling/chan-end-run! run-id))))
-(declare activate-table-and-mark-computed!)
+(declare activate-table-and-mark-computed! target-table)
(defn sync-target!
"Sync target of a transform"
@@ -198,19 +203,132 @@
[query]
(assoc-in query [:middleware :disable-remaps?] true))
-(defn compile-source
- "Compile the source query of a transform."
- [{query-type :type :as source}]
- (case (keyword query-type)
- :query (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
+(defn- checkpoint-incremental?
+ "Returns true if `source` uses checkpoint-based incremental strategy."
+ [source]
+ (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
+
+(defn- source->checkpoint-filter-unique-key
+ "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
+ [query source-incremental-strategy]
+ (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))
+
+(defn- source->checkpoint-filter-column
+ "Resolve the checkpoint filter column for an incremental transform.
+
+ Tries to resolve the column using the unique key first.
+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."
+ [query source-incremental-strategy table metadata-provider]
+ (or
+ (source->checkpoint-filter-unique-key query source-incremental-strategy)
+ (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]
+ (when-let [field-id (t2/select-one-pk :model/Field
+ :table_id (:id table)
+ :name field-name)]
+ (lib.metadata/field metadata-provider field-id)))))
+
+(defn next-checkpoint
+ "Build a query to compute the MAX of the checkpoint column from the target table.
+
+ Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),
+ or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."
+ [transform-id]
+ (let [{:keys [source target] :as transform} (t2/select-one :model/Transform transform-id)
+ db-id (transforms.i/target-db-id transform)]
+ (when (checkpoint-incremental? source)
+ (when-let [table (target-table db-id target)]
+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
+ table-metadata (lib.metadata/table metadata-provider (:id table))
+ query (lib/query metadata-provider table-metadata)]
+ (when-let [filter-column (source->checkpoint-filter-column query
+ (:source-incremental-strategy source)
+ table metadata-provider)]
+ {:query (-> query (lib/aggregate (lib/max filter-column)))
+ :filter-column filter-column}))))))
+
+(defn- next-checkpoint-value
+ "Execute the checkpoint query and normalize the result for database insertion.
+ Returns `nil` if the target table is empty."
+ [{:keys [query filter-column]}]
+ (let [{:keys [base-type]} filter-column
+ v (some-> query qp/process-query :data :rows first first)]
+ ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right
+ ;; shape for reinsertion
+ (cond
+ (nil? v)
+ nil
+
+ (isa? base-type :type/Integer)
+ (bigint v)
+
+ ;; any other number that's not an integer, should be a decimal/float
+ (number? v)
+ (bigdec v)
+
+ :else v)))
+
+(defn preprocess-incremental-query
+ "Add checkpoint checkpoint filtering to a query for incremental execution.
+
+ For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.
+ For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.
+ Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."
+ [query source-incremental-strategy checkpoint]
+ (if-let [checkpoint-value (next-checkpoint-value checkpoint)]
+ (if (lib.query/native? query)
+ ;; native query with explicit checkpoint filter
+ (if (get-in query [:stages 0 :template-tags "checkpoint"])
+ (update query :parameters conj
+ {:type (if (number? checkpoint-value) :number :text)
+ :target [:variable [:template-tag "checkpoint"]]
+ :value checkpoint-value})
+ query)
+ ;; mbql query
+ (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))
+ query))
+
+(defn- post-process-incremental-query
+ "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.
+
+ Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "
+ [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]
+ (let [{:keys [checkpoint-filter]} source-incremental-strategy]
+ (if (and (lib.query/native? (:query source))
+ (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))
+ (next-checkpoint-value checkpoint))
+ (let [wrap-query (fn [query]
+ (let [honeysql-query {:select [:*]
+ :from [[[:raw (str "(" query ")")] :subquery]]
+ :where [:> (h2x/identifier :field checkpoint-filter)
+ [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]
+ (first (sql.qp/format-honeysql driver honeysql-query))))]
+ (update outer-query :query wrap-query))
+ outer-query)))
-(defn required-database-feature
- "Returns the database feature necessary to execute `transform`."
+(defn compile-source
+ "Compile the source query of a transform to SQL, applying incremental filtering if required."
+ [{:keys [id source]}]
+ (let [{:keys [source-incremental-strategy] query-type :type} source]
+ (case (keyword query-type)
+ :query
+ (let [checkpoint (next-checkpoint id)
+ query (:query source)
+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
+ (binding [driver/*compile-with-inline-parameters*
+ (or (= :clickhouse driver)
+ driver/*compile-with-inline-parameters*)]
+ (-> query
+ (preprocess-incremental-query source-incremental-strategy checkpoint)
+ massage-sql-query
+ qp.compile/compile
+ (post-process-incremental-query driver source checkpoint)))))))
+
+(defn required-database-features
+ "Returns the database features necessary to execute `transform`."
[transform]
(if (python-transform? transform)
- :transforms/python
- (case (-> transform :target :type)
- "table" :transforms/table)))
+ [:transforms/python]
+ [:transforms/table]))
(defn ->instant
"Convert a temporal value `t` to an Instant in the system timezone."
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
index 79a807c3a2c..c41c7552c9a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
@@ -64,7 +64,7 @@
(with-open [shared-storage-ref (s3/open-shared-storage! source_tables)]
(let [server-url (transforms-python.settings/python-runner-url)
_ (python-runner/copy-tables-to-s3! {:shared-storage @shared-storage-ref
- :table-name->id source_tables
+ :source {:source-tables source_tables}
:limit per_input_row_limit})
{runner-status :status
runner-body :body}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 75a6338559f..cabcd70d5da 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -140,22 +140,28 @@
ret)))
(thunk)))
-(defn- create-table-and-insert-data!
- "Create a table from metadata and insert data from source."
- [driver db-id table-name metadata data-source]
- (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
- :columns (mapv (fn [{:keys [name base_type #_database_type]}]
- {:name name
- :type (python-runner/restricted-insert-type base_type)
- ;; :database-type database_type
- :nullable? true})
- (:fields metadata))}
- data-source (assoc data-source :table-schema table-schema)]
- (transforms.util/create-table-from-schema! driver db-id table-schema)
-
+(defn- table-schema [table-name metadata]
+ {:name (if (keyword? table-name) table-name (keyword table-name))
+ :columns (mapv (fn [{:keys [name base_type #_database_type]}]
+ {:name name
+ :type (python-runner/restricted-insert-type base_type)
+ ;; :database-type database_type
+ :nullable? true})
+ (:fields metadata))})
+
+(defn- insert-data!
+ "Insert data from source into an existing table."
+ [driver db-id table-schema data-source]
+ (let [data-source (assoc data-source :table-schema table-schema)]
(->> #(driver/insert-from-source! driver db-id table-schema data-source)
(maybe-retry-with-backoff driver))))
+(defn- create-table-and-insert-data!
+ "Create a table from metadata and insert data from source."
+ [driver db-id table-schema data-source]
+ (transforms.util/create-table-from-schema! driver db-id table-schema)
+ (insert-data! driver db-id table-schema data-source))
+
(defn- transfer-with-rename-tables-strategy!
"Transfer data using the rename-tables*! multimethod with atomicity guarantees.
Creates new table, then atomically renames target->old and new->target, then drops old."
@@ -168,7 +174,7 @@
(log/info "Using rename-tables strategy with atomicity guarantees")
(try
- (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/rename-tables! driver db-id {table-name temp-table-name
source-table-name table-name})
(transforms.util/drop-table! driver db-id temp-table-name)
@@ -188,7 +194,7 @@
(log/info "Using create-drop-rename strategy to minimize downtime")
(try
- (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/drop-table! driver db-id table-name)
(driver/rename-table! driver db-id source-table-name table-name)
@@ -207,13 +213,37 @@
(try
(transforms.util/drop-table! driver db-id table-name)
- (create-table-and-insert-data! driver db-id table-name metadata data-source)
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
(catch Exception e
(log/error e "Failed to transfer data using drop-create fallback strategy")
(throw e))))
-(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]
+(defmulti ^:private transfer-file-to-db
+ {:arglists '([driver db transform metadata temp-file])}
+ (fn [_ _ transform _ _] (-> transform :target :type keyword)))
+
+(defmethod transfer-file-to-db :table-incremental
+ [driver {db-id :id}
+ {:keys [target] :as transform}
+ metadata temp-file]
+ (let [table-name (transforms.util/qualified-table-name driver target)
+ table-exists? (transforms.util/target-table-exists? transform)
+ data-source {:type :jsonl-file
+ :file temp-file}]
+
+ ;; once we have more than just append, dispatch on :target-incremental-strategy
+
+ (if (not table-exists?)
+ (do
+ (log/info "New table")
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+ (insert-data! driver db-id (table-schema table-name metadata) data-source))))
+
+(defmethod transfer-file-to-db :table
+ [driver {db-id :id :as db}
+ {:keys [target] :as transform}
+ metadata temp-file]
(let [table-name (transforms.util/qualified-table-name driver target)
table-exists? (transforms.util/target-table-exists? transform)
data-source {:type :jsonl-file
@@ -222,7 +252,7 @@
(not table-exists?)
(do
(log/info "New table")
- (create-table-and-insert-data! driver db-id table-name metadata data-source))
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
(driver.u/supports? driver :atomic-renames db)
(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
@@ -253,10 +283,11 @@
server-url (transforms-python.settings/python-runner-url)
_ (python-runner/copy-tables-to-s3! {:run-id run-id
:shared-storage @shared-storage-ref
- :table-name->id (:source-tables source)
- :cancel-chan cancel-chan})
+ :source source
+ :cancel-chan cancel-chan
+ :limit (:limit source)
+ :transform-id (:id transform)})
_ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
-
{:keys [status body] :as response}
(python-runner/execute-python-code-http-call!
{:server-url server-url
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index c12c1542741..c7c97850881 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -7,8 +7,12 @@
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase-enterprise.transforms.util :as transforms.u]
[metabase.analytics.prometheus :as prometheus]
[metabase.config.core :as config]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.util.i18n :as i18n]
@@ -81,8 +85,7 @@
`maybe-fixup-value` before JSON encoding."
[^OutputStream os fields-meta {cols-meta :cols}]
(let [filtered-col-meta (m/index-by :name fields-meta)
- col-names (map :name cols-meta)
- none? (volatile! true)]
+ col-names (map :name cols-meta)]
(fn
([]
(-> os
@@ -90,9 +93,6 @@
(BufferedWriter.)))
([^BufferedWriter writer]
- ;; Workaround for LocalStack, which doesn't support zero byte files.
- (when @none?
- (.write writer " "))
(doto writer
(.flush)
(.close)))
@@ -105,32 +105,31 @@
(map (fn [[n v]]
(maybe-fixup-value (filtered-col-meta n) v)))
(zipmap (filter filtered-col-meta col-names)))]
- (when @none? (vreset! none? false))
(json/encode-to row-map writer {})
(doto writer
(.newLine)))))))
(defn- execute-mbql-query
- [db-id query rff cancel-chan]
+ [query rff cancel-chan]
;; if we have a cancel-chan (a promise channel) for the transform, we'd like for QP to respect it
;; and early exit if a value is delivered, but QP closes it when it's done. So we copy it.
(with-bindings* (cond-> {}
cancel-chan
(assoc #'qp.pipeline/*canceled-chan* (a/go (a/<! cancel-chan))))
(^:once fn* []
- (qp/process-query {:type :query :database db-id :query query} rff))))
+ (qp/process-query query rff))))
(defn- throw-if-cancelled [cancel-chan]
(when (a/poll! cancel-chan)
(throw (ex-info "Run cancelled" {:error-type :cancelled}))))
-(defn- write-table-data-to-file! [{:keys [db-id table-id fields-meta temp-file cancel-chan limit]}]
+(defn- write-query-data-to-file! [{:keys [query fields-meta temp-file cancel-chan]}]
(with-open [os (io/output-stream temp-file)]
- (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))
- rff (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))]
- (execute-mbql-query db-id query rff cancel-chan)
- (some-> cancel-chan throw-if-cancelled)
- nil)))
+ (execute-mbql-query query
+ (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))
+ cancel-chan)
+ (some-> cancel-chan throw-if-cancelled)
+ nil))
(defn restricted-insert-type
"Type for insertion restricted to supported"
@@ -267,17 +266,31 @@
:nfc_path nil
{:order-by [[:database_position :asc]]}))
+(defn- build-table-query
+ "Build a mbql query for table, might add a proper filter for incremental transforms."
+ [table-id source-incremental-strategy transform-id limit]
+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+ metadata-provider (lib-be/application-database-metadata-provider db-id)
+ table-metadata (lib.metadata/table metadata-provider table-id)]
+ (cond-> (lib/query metadata-provider table-metadata)
+ source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform-id))
+ limit (lib/limit limit))))
+
;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
(defn copy-tables-to-s3!
"Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
Blocks until all tables are fully written and committed to shared storage."
[{:keys [run-id
shared-storage
- table-name->id
+ source
cancel-chan
- limit]}]
+ limit
+ transform-id]}]
+ (when (and (:source-incremental-strategy source)
+ (> (count (:source-tables source)) 1))
+ (throw (ex-info "Incremental transforms for python only supports one source table" {})))
;; TODO there's scope for some parallelism here, in particular across different databases
- (doseq [[table-name table-id] table-name->id
+ (doseq [[table-name table-id] (:source-tables source)
:let [{:keys [s3-client bucket-name objects]} shared-storage
{data-path :path} (get objects [:table table-id :data])
{manifest-path :path} (get objects [:table table-id :manifest])]]
@@ -289,14 +302,12 @@
fields-meta (fields-metadata driver table-id)
manifest (generate-manifest table-id fields-meta)]
(transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]
- (write-table-data-to-file!
- {:db-id db-id
- :driver driver
- :table-id table-id
- :fields-meta fields-meta
- :temp-file tmp-data-file
- :cancel-chan cancel-chan
- :limit limit}))
+ (let [query (build-table-query table-id (:source-incremental-strategy source) transform-id limit)]
+ (write-query-data-to-file!
+ {:query query
+ :fields-meta fields-meta
+ :temp-file tmp-data-file
+ :cancel-chan cancel-chan})))
(with-open [writer (io/writer tmp-meta-file)]
(json/encode-to manifest writer {}))
(let [data-size (.length tmp-data-file)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
index 22cc6c01523..e244fab72b1 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
@@ -179,9 +179,12 @@
(defn upload-file
"Upload the given file to s3"
- [^S3Client s3-client ^String bucket-name ^String key ^File file]
- (let [^PutObjectRequest request (put-object-request bucket-name key)]
- (.putObject s3-client request (RequestBody/fromFile file))))
+ [^S3Client s3 ^String bucket ^String key ^File file]
+ (let [^PutObjectRequest req (put-object-request bucket key)
+ body (if (zero? (.length file))
+ (RequestBody/empty)
+ (RequestBody/fromFile file))]
+ (.putObject s3 req body)))
(defn open-object
"Get back the contents of the given key as a InputStream."
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index 6abfba61e9e..fb632bcdf48 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -912,3 +912,85 @@
:schema (get-test-schema)
:name table-name}}]
(test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))
+
+(deftest ^:parallel extract-columns-from-query-test
+ (testing "POST /api/ee/transform/extract-columns"
+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/table)
+ :clickhouse :redshift :bigquery-cloud-sdk)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (letfn [(make-native-query [sql]
+ {:lib/type :mbql/query
+ :database (mt/id)
+ :stages [{:lib/type :mbql.stage/native
+ :native sql}]})]
+ (testing "Successfully extracts columns from a simple SELECT query"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
+ {:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]
+ (is (= ["id" "name" "category" "price"] (:columns response)))))
+
+ (testing "Returns nil for invalid SQL"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
+ {:query (make-native-query "SELECT * FORM invalid_table")})]
+ (is (nil? (:columns response)))))
+
+ (testing "Extracts columns from query with aliases"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
+ {:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]
+ (is (= ["product_id" "product_name"] (:columns response)))))
+
+ (testing "Requires superuser permissions"
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :post 403 "ee/transform/extract-columns"
+ {:query (make-native-query "SELECT * FROM transforms_products")}))))
+
+ (testing "Returns 404 for non-existent database"
+ (is (= "Not found."
+ (mt/user-http-request :crowberto :post 404 "ee/transform/extract-columns"
+ {:query {:lib/type :mbql/query
+ :database 999999
+ :stages [{:lib/type :mbql.stage/native
+ :native "SELECT * FROM transforms_products"}]}}))))))))))
+
+(deftest ^:parallel is-simple-query-test
+ (testing "POST /api/ee/transform/is-simple-query"
+ (mt/with-premium-features #{:transforms}
+ (testing "Returns true for simple SELECT queries"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT id, name FROM products"})]
+ (is (true? (:is_simple response)))
+ (is (nil? (:reason response)))))
+
+ (testing "Returns true for simple SELECT with WHERE clause"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT id, name FROM products WHERE category = 'Electronics'"})]
+ (is (true? (:is_simple response)))))
+
+ (testing "Returns true for simple SELECT with JOIN"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT p.id, p.name, c.name FROM products p JOIN categories c ON p.category_id = c.id"})]
+ (is (true? (:is_simple response)))))
+
+ (testing "Returns false for query with LIMIT"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT id, name FROM products LIMIT 10"})]
+ (is (false? (:is_simple response)))
+ (is (= "Contains a LIMIT" (:reason response)))))
+
+ (testing "Returns false for query with OFFSET"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT id, name FROM products OFFSET 5"})]
+ (is (false? (:is_simple response)))
+ (is (= "Contains an OFFSET" (:reason response)))))
+
+ (testing "Returns false for query with LIMIT and OFFSET"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "SELECT id, name FROM products LIMIT 10 OFFSET 5"})]
+ (is (false? (:is_simple response)))
+ (is (= "Contains a LIMIT" (:reason response)))))
+
+ (testing "Returns false for query with CTE"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"
+ {:query "WITH category_counts AS (SELECT category, COUNT(*) as cnt FROM products GROUP BY category) SELECT * FROM category_counts"})]
+ (is (false? (:is_simple response)))
+ (is (= "Contains a CTE" (:reason response))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
new file mode 100644
index 00000000000..9cdef5a6fbc
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -0,0 +1,554 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms.incremental-test
+ "Tests for incremental transforms functionality."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [honey.sql :as sql]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms.interface :as transforms.i]
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [metabase-enterprise.transforms.util :as transforms.u]
+ [metabase.driver :as driver]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.util :as sql.u]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor :as qp]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(def checkpoint-configs
+ {:integer {:field-name "id"
+ :template-tag-type :number
+ :lib-column-key "column-unique-key-v1$id"
+ :expected-initial-checkpoint 10
+ :expected-second-checkpoint 16}
+ :float {:field-name "price"
+ :template-tag-type :number
+ :lib-column-key "column-unique-key-v1$price"
+ :expected-initial-checkpoint 49.99
+ :expected-second-checkpoint 199.99}
+ :temporal {:field-name "created_at"
+ :template-tag-type :text
+ :lib-column-key "column-unique-key-v1$created_at"
+ :expected-initial-checkpoint "2024-01-10"
+ :expected-second-checkpoint "2024-01-16"}})
+
+(defn- valid-checkpoint-transform-combo?
+ "Check if a checkpoint type and transform type combination is valid. "
+ [checkpoint-type transform-type]
+ (not (and (= checkpoint-type :temporal)
+ (= transform-type :native))))
+
+(defn- build-table-query-with-order-by
+ "Wraps the original build-table-query to add ORDER BY for deterministic test results."
+ [original-fn table-id source-incremental-strategy transform-id limit checkpoint-field]
+ (let [query (original-fn table-id source-incremental-strategy transform-id limit)
+ db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+ metadata-provider (lib-be/application-database-metadata-provider db-id)
+ ;; Find the checkpoint field in the table metadata
+ checkpoint-col (->> (lib.metadata/fields metadata-provider table-id)
+ (filter #(= (:name %) checkpoint-field))
+ first)]
+ (if checkpoint-col
+ (lib/order-by query checkpoint-col)
+ query)))
+
+(defmacro with-python-order-by!
+ "Execute body with python-runner/build-table-query redef'd to include ORDER BY for deterministic test results."
+ [checkpoint-field & body]
+ `(let [original-fn# @#'python-runner/build-table-query]
+ (with-redefs [python-runner/build-table-query
+ (fn [table-id# source-incremental-strategy# transform-id# limit#]
+ (build-table-query-with-order-by original-fn# table-id# source-incremental-strategy# transform-id# limit# ~checkpoint-field))]
+ ~@body)))
+
+(defn- execute-transform-with-ordering!
+ "Execute a transform, wrapping Python transforms with ORDER BY redefinition for deterministic results."
+ [transform transform-type checkpoint-field run-opts]
+ (if (= transform-type :python)
+ (with-python-order-by! checkpoint-field
+ (transforms.i/execute! transform run-opts))
+ (transforms.i/execute! transform run-opts)))
+
+(defn- make-incremental-source-query
+ "Create a native query with optional checkpoint template tag."
+ [schema checkpoint-config]
+ (let [{:keys [field-name template-tag-type]} checkpoint-config
+ timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
+ query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"
+ timestamp-sql
+ (if schema
+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")
+ "transforms_products")
+ field-name field-name)]
+ {:database (mt/id)
+ :type :native
+ :native {:query query
+ :template-tags {"checkpoint" {:id "checkpoint"
+ :name "checkpoint"
+ :display-name "Checkpoint"
+ :type template-tag-type
+ :required false}}}}))
+
+(defn- make-incremental-source-query-without-template-tag
+ "Create a native query without template tags for testing automatic checkpoint insertion. "
+ [schema]
+ (let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
+ query (format "SELECT *, %s AS load_timestamp FROM %s"
+ timestamp-sql
+ (if schema
+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")
+ "transforms_products"))]
+ {:database (mt/id)
+ :type :native
+ :native {:query query}}))
+
+(defn- make-incremental-mbql-query
+ "Create an MBQL query for incremental transforms. "
+ [checkpoint-config]
+ (let [{:keys [field-name]} checkpoint-config]
+ (mt/mbql-query transforms_products {:expressions {"load_timestamp" [:now]}
+ :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]
+ :limit 10})))
+
+(def incremental-python-body
+ (str "import pandas as pd\n"
+ "from datetime import datetime\n"
+ "\n"
+ "def transform(transforms_products):\n"
+ " transforms_products['load_timestamp'] = datetime.now()\n"
+ " return transforms_products"))
+
+(defn- make-incremental-transform-payload
+ "Create a transform payload for incremental transform testing.
+
+ transform-type can be :native, :native-auto-wrap, :mbql, or :python
+ checkpoint-config should be from checkpoint-configs"
+ [transform-name target-table-name transform-type checkpoint-config]
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ {:keys [field-name lib-column-key]} checkpoint-config]
+ {:name transform-name
+ :source (case transform-type
+ :native {:type "query"
+ :query (make-incremental-source-query schema checkpoint-config)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter field-name}}
+ :native-auto-wrap {:type "query"
+ :query (make-incremental-source-query-without-template-tag schema)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter field-name}}
+ :mbql {:type "query"
+ :query (make-incremental-mbql-query checkpoint-config)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-unique-key lib-column-key}}
+ :python {:type "python"
+ :source-tables {"transforms_products" (mt/id :transforms_products)}
+ :limit 10
+ :body incremental-python-body
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-unique-key lib-column-key}})
+ :target {:type "table-incremental"
+ :schema schema
+ :name target-table-name
+ :database (mt/id)
+ :target-incremental-strategy {:type "append"}}}))
+
+(defn- get-table-row-count
+ "Get the row count of a table by name."
+ [table-name]
+ (let [table (t2/select-one :model/Table :name table-name)
+ mp (mt/metadata-provider)
+ table-metadata (lib.metadata/table mp (:id table))
+ count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))
+ result (qp/process-query count-query)]
+ (some-> result :data :rows first first bigint)))
+
+(defn- get-distinct-timestamp-count
+ "Get the count of distinct load_timestamp values in a table."
+ [table-name]
+ (let [table (t2/select-one :model/Table :name table-name)
+ native-query {:database (mt/id)
+ :type :native
+ :native {:query (format "SELECT COUNT(DISTINCT load_timestamp) FROM %s"
+ (sql.u/quote-name driver/*driver* :table
+ (:schema table)
+ table-name))}}
+ result (qp/process-query native-query)]
+ (some-> result :data :rows first first bigint)))
+
+(defn get-checkpoint-value [transform-id]
+ (#'transforms.u/next-checkpoint-value
+ (transforms.u/next-checkpoint transform-id)))
+
+(defn- compare-checkpoint-values
+ "Compare two checkpoint values with type-appropriate logic. "
+ [checkpoint-type expected actual]
+ (case checkpoint-type
+ :integer (= expected actual)
+ :float (and (number? actual)
+ (< (Math/abs (- expected actual)) 0.01))
+ :temporal (and (string? actual)
+ (str/starts-with? actual expected))))
+
+(defn- insert-test-products!
+ "Insert new products into the transforms_products table."
+ [products]
+ (let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ schema-prefix (if schema (str schema ".") "")
+ values-list (str/join ", "
+ (map (fn [{:keys [name category price created-at]}]
+ (format "('%s', '%s', %s, '%s')" name category price created-at))
+ products))
+ insert-sql (format "INSERT INTO %s%s (name, category, price, created_at) VALUES %s"
+ schema-prefix
+ source-table-name
+ values-list)]
+ (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))
+
+(defn- delete-test-products!
+ [products]
+ (let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ schema-prefix (if schema (str schema ".") "")
+ delete-sql (format "DELETE FROM %s%s WHERE name IN (%s)"
+ schema-prefix
+ source-table-name
+ (str/join ", " (map (constantly "?") products)))]
+ (driver/execute-raw-queries! driver/*driver* spec [[delete-sql (mapv :name products)]])))
+
+(defmacro with-insert-test-products! [products & body]
+ `(let [products# ~products]
+ (try
+ (insert-test-products! products#)
+ ~@body
+ (finally
+ (delete-test-products! products#)))))
+
+(set! *warn-on-reflection* true)
+
+(defn- test-drivers []
+ (disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))
+
+(deftest create-incremental-transform-test
+ (testing "Creating an incremental transform with checkpoint strategy"
+ (doseq [checkpoint-type [:integer :float]
+ :let [transform-type :native]]
+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "incremental_test"]
+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)
+ {:keys [field-name]} checkpoint-config
+ transform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)]
+ (testing "Transform is created successfully"
+ (mt/with-temp [:model/Transform transform transform-payload]
+ (is (some? (:id transform)))
+ (is (= "Test Incremental Transform" (:name transform)))
+ (is (= "table-incremental" (-> transform :target :type)))
+ (is (= "checkpoint" (-> transform :source :source-incremental-strategy :type)))
+ (is (= field-name (-> transform :source :source-incremental-strategy :checkpoint-filter)))
+
+ (testing "No checkpoint exists initially"
+ (is (nil? (get-checkpoint-value (:id transform)))))
+
+ (testing "Can retrieve transform via API"
+ (let [retrieved (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" (:id transform)))]
+ (is (= (:id transform) (:id retrieved)))
+ (is (= "Test Incremental Transform" (:name retrieved)))))
+
+ (testing "Transform appears in list endpoint"
+ (let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
+ our-transform (first (filter #(= (:id transform) (:id %)) transforms))]
+ (is (some? our-transform))
+ (is (= "Test Incremental Transform" (:name our-transform))))))))))))))))
+
+(deftest run-incremental-transform-twice-test
+ (testing "Running an incremental transform twice processes only new data on second run"
+ (doseq [checkpoint-type [:integer :float :temporal]
+ transform-type [:native :mbql :python]
+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "incremental_twice"]
+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)
+ {:keys [expected-initial-checkpoint expected-second-checkpoint]} checkpoint-config
+ transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]
+ (mt/with-temp [:model/Transform transform transform-payload]
+ (testing "First run processes all data"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (transforms.tu/wait-for-table target-table 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 10 row-count) "First run should process the first 10 products")
+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
+
+ (testing "Checkpoint is created after first run"
+ (let [checkpoint (get-checkpoint-value (:id transform))]
+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from first 10 rows" (:field-name checkpoint-config)))))))
+
+ (testing "Second run should process the remaining rows"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 16 row-count) "Second run should add remaining 6 rows")
+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")
+ (let [checkpoint (get-checkpoint-value (:id transform))]
+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config))))))))))))))))
+
+(deftest switch-incremental-to-non-incremental-test
+ (testing "Switching an incremental transform to non-incremental overwrites data"
+ (doseq [checkpoint-type [:integer :float :temporal]
+ transform-type [:native :mbql :python]
+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "switch_incr_to_non_incr"]
+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)
+ {:keys [expected-initial-checkpoint]} checkpoint-config
+ initial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)]
+ (mt/with-temp [:model/Transform transform initial-payload]
+ (testing "Initial incremental run processes first batch"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (transforms.tu/wait-for-table target-table 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 10 row-count) "Initial run should process first 10 products")
+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))
+
+ (testing "Second incremental run processes remaining data"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 16 row-count) "Should have 16 rows after second incremental run")
+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")))
+
+ (testing "Switch to non-incremental via PUT API"
+ (let [non-incremental-payload (-> initial-payload
+ (update :source dissoc :source-incremental-strategy)
+ (update :target dissoc :source-incremental-strategy)
+ (update :target assoc :type "table"))
+ updated (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" (:id transform))
+ non-incremental-payload)]
+ (is (= "table" (-> updated :target :type)))
+ (is (nil? (-> updated :source :source-incremental-strategy)))))
+
+ (testing "Non-incremental run overwrites all data"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 10 row-count) "Should overwrite to 10 rows")
+ (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))
+
+ (testing "Running again still overwrites"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 10 row-count) "Should still have 10 rows after another run")
+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))))))))))))))
+
+(deftest switch-non-incremental-to-incremental-test
+ (testing "Switching a non-incremental transform to incremental computes checkpoint from existing data"
+ (doseq [checkpoint-type [:integer :float :temporal]
+ transform-type [:native :mbql :python]
+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "switch_non_incr_to_incr"]
+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)
+ {:keys [expected-second-checkpoint]} checkpoint-config
+ incremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)
+ initial-payload (-> incremental-payload
+ (update :source dissoc :source-incremental-strategy)
+ (update :target dissoc :source-incremental-strategy)
+ (update :target assoc :type "table"))]
+ (mt/with-temp [:model/Transform transform initial-payload]
+ (testing "Initial non-incremental run creates table"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (transforms.tu/wait-for-table target-table 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 10 row-count) "Initial run should process 10 products")
+ (is (= 1 distinct-timestamps) "All rows should have same timestamp from non-incremental run")
+
+ (testing "No checkpoint exists"
+ (let [checkpoint (get-checkpoint-value (:id transform))]
+ (is (nil? checkpoint) "No checkpoint for non-incremental transform")))))
+
+ (testing "Switch to incremental via PUT API"
+ (let [updated (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" (:id transform))
+ incremental-payload)]
+ (is (= "table-incremental" (-> updated :target :type)))
+ (is (= "checkpoint" (-> updated :source :source-incremental-strategy :type)))))
+
+ (testing "First incremental run after switch processes no new data"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "Should process remaining 6 entries")
+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")
+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))
+
+ (when-not (= driver/*driver* :clickhouse)
+ (testing "Add new data and run incrementally"
+ (with-insert-test-products!
+ [{:name "After Switch Product"
+ :category "Gadget"
+ :price 379.99
+ :created-at "2024-01-20T10:00:00"}]
+
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
+ ;; For integer checkpoints, we can verify exact value >= 17
+ ;; For float/temporal, just verify checkpoint exists
+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))
+
+(deftest native-query-without-template-tag-test
+ (testing "Native query without template tags uses automatic checkpoint wrapping"
+ (doseq [checkpoint-type [:integer :float :temporal]]
+ (testing (format "with %s checkpoint" (name checkpoint-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "native_no_template"]
+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)
+ {:keys [expected-second-checkpoint field-name]} checkpoint-config
+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ transform-payload {:name "Native Without Template Tag"
+ :source {:type "query"
+ :query (make-incremental-source-query-without-template-tag schema)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter field-name}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name target-table
+ :database (mt/id)
+ :target-incremental-strategy {:type "append"}}}]
+ (mt/with-temp [:model/Transform transform transform-payload]
+ (testing "First run processes all existing data"
+ (transforms.i/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table target-table 10000)
+ (let [row-count (get-table-row-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "First run should process all 16 products")
+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))
+
+ (testing "Second run without new data adds nothing"
+ (transforms.i/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows, no new data")))
+
+ (when-not (= driver/*driver* :clickhouse)
+ (testing "After inserting new data, incremental run appends only new rows"
+ (with-insert-test-products!
+ [{:name "New Product 1"
+ :category "Electronics"
+ :price 299.99
+ :created-at "2024-01-21T10:00:00"}
+ {:name "New Product 2"
+ :category "Books"
+ :price 319.99
+ :created-at "2024-01-21T11:00:00"}]
+
+ (transforms.i/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")
+ (is (some? checkpoint) "Checkpoint should be updated")))))))))))))))
+
+(deftest ^:postgres-only native-query-with-temporal-checkpoint-test
+ (testing "Native query with temporal checkpoint"
+ ;; we test only in postgres because it's easy to cast to ::timestamp
+ (mt/test-drivers [:postgres]
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "native_temporal_cast"]
+ (let [checkpoint-config (get checkpoint-configs :temporal)
+ {:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-config
+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
+ query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}::timestamp]] ORDER BY %s LIMIT 10"
+ timestamp-sql
+ (if schema
+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")
+ "transforms_products")
+ field-name
+ field-name)
+ transform-payload {:name "Native With Temporal Cast"
+ :source {:type "query"
+ :query {:database (mt/id)
+ :type :native
+ :native {:query query
+ :template-tags {"checkpoint" {:id "checkpoint"
+ :name "checkpoint"
+ :display-name "Checkpoint"
+ :type :text
+ :required false}}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter field-name}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name target-table
+ :database (mt/id)
+ :target-incremental-strategy {:type "append"}}}]
+ (mt/with-temp [:model/Transform transform transform-payload]
+ (testing "First run processes first batch"
+ (transforms.i/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table target-table 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 10 row-count) "First run should process the first 10 products")
+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
+ (is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))
+
+ (testing "Second run processes remaining data"
+ (transforms.i/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "Second run should add remaining 6 rows")
+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")
+ (is (compare-checkpoint-values :temporal expected-second-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))
+
+ (testing "Third run without new data adds nothing"
+ (transforms.i/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows, no new data")))
+
+ (testing "After inserting new data, incremental run appends only new rows"
+ (with-insert-test-products!
+ [{:name "New Temporal Product"
+ :category "Electronics"
+ :price 299.99
+ :created-at "2024-01-21T10:00:00"}]
+
+ (transforms.i/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index b36ea5efb50..1761bacf0ed 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -1,13 +1,11 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.drivers-test
"Comprehensive tests for Python transforms across all supported drivers with all base and exotic types."
(:require
- [clojure.core.async :as a]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
- [metabase-enterprise.transforms-python.s3 :as s3]
- [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms-python.python-runner-test :as python-runner-test]
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
@@ -23,32 +21,6 @@
(set! *warn-on-reflection* true)
-(def test-id 42)
-
-(defn- execute!
- "Execute a Python transform with the given code and tables"
- [{:keys [code tables]}]
- (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
- (let [server-url (transforms-python.settings/python-runner-url)
- cancel-chan (a/promise-chan)
- table-name->id (or tables {})
- _ (python-runner/copy-tables-to-s3! {:run-id test-id
- :shared-storage @shared-storage-ref
- :table-name->id table-name->id
- :cancel-chan cancel-chan})
- response (python-runner/execute-python-code-http-call! {:server-url server-url
- :code code
- :run-id test-id
- :table-name->id table-name->id
- :shared-storage @shared-storage-ref})
- events (python-runner/read-events @shared-storage-ref)
- output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
- (merge (:body response)
- {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
- :output-manifest output-manifest
- :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
- :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
-
(defn- execute-e2e-transform!
"Execute an e2e Python transform test using execute-python-transform!"
[table-name transform-code source-tables]
@@ -245,8 +217,8 @@
(defn- execute-and-validate-transform!
"Execute a Python transform and validate the output, returning validation results."
[transform-code table-name table-id expected-columns expected-row-count]
- (let [result (execute! {:code transform-code
- :tables {table-name table-id}})]
+ (let [result (python-runner-test/execute! {:code transform-code
+ :tables {table-name table-id}})]
(validate-transform-output result expected-columns expected-row-count)))
(defn- test-exotic-types-for-driver!
@@ -409,11 +381,11 @@
" df['name_upper'] = df['name'].str.upper()\n"
" return df")
;; Run the transform twice
- result1 (execute!
+ result1 (python-runner-test/execute!
{:code transform-code
:tables {table-name table-id}})
- result2 (execute!
+ result2 (python-runner-test/execute!
{:code transform-code
:tables {table-name table-id}})]
(testing "Both transforms succeeded"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index c542983351c..dbc045a95d9 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -55,7 +55,7 @@
(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))
-(defn- execute! [{:keys [code tables]}]
+(defn execute! [{:keys [code tables]}]
(with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
(let [server-url (transforms-python.settings/python-runner-url)
cancel-chan (a/promise-chan)
@@ -63,7 +63,7 @@
test-id (next-job-run-id)
_ (python-runner/copy-tables-to-s3! {:run-id test-id
:shared-storage @shared-storage-ref
- :table-name->id table-name->id
+ :source {:source-tables table-name->id}
:cancel-chan cancel-chan})
response (python-runner/execute-python-code-http-call! {:server-url server-url
:code code
diff --git a/enterprise/frontend/src/metabase-enterprise/api/transform.ts b/enterprise/frontend/src/metabase-enterprise/api/transform.ts
index 45345abca88..92e29222e4a 100644
--- a/enterprise/frontend/src/metabase-enterprise/api/transform.ts
+++ b/enterprise/frontend/src/metabase-enterprise/api/transform.ts
@@ -1,6 +1,10 @@
import { isResourceNotFoundError } from "metabase/lib/errors";
import type {
+ CheckQueryComplexityRequest,
+ CheckQueryComplexityResponse,
CreateTransformRequest,
+ ExtractColumnsFromQueryRequest,
+ ExtractColumnsFromQueryResponse,
ListTransformRunsRequest,
ListTransformRunsResponse,
ListTransformsRequest,
@@ -189,6 +193,26 @@ export const transformApi = EnterpriseApi.injectEndpoints({
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("transform"), listTag("table")]),
}),
+ extractColumnsFromQuery: builder.mutation<
+ ExtractColumnsFromQueryResponse,
+ ExtractColumnsFromQueryRequest
+ >({
+ query: (body) => ({
+ method: "POST",
+ url: "/api/ee/transform/extract-columns",
+ body,
+ }),
+ }),
+ checkQueryComplexity: builder.query<
+ CheckQueryComplexityResponse,
+ CheckQueryComplexityRequest
+ >({
+ query: (queryString) => ({
+ method: "POST",
+ url: "/api/ee/transform/is-simple-query",
+ body: { query: queryString },
+ }),
+ }),
}),
});
@@ -204,4 +228,6 @@ export const {
useUpdateTransformMutation,
useDeleteTransformMutation,
useDeleteTransformTargetMutation,
+ useExtractColumnsFromQueryMutation,
+ useLazyCheckQueryComplexityQuery,
} = transformApi;
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
new file mode 100644
index 00000000000..21ce0e2bb48
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -0,0 +1,256 @@
+import { useFormikContext } from "formik";
+import { useEffect, useMemo, useState } from "react";
+import { t } from "ttag";
+
+import { FormSelect, FormSwitch } from "metabase/forms";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
+import { Alert, Stack } from "metabase/ui";
+import { useLazyCheckQueryComplexityQuery } from "metabase-enterprise/api";
+import {
+ KeysetColumnSelect,
+ PythonKeysetColumnSelect,
+} from "metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect";
+import { NativeQueryColumnSelect } from "metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect";
+import {
+ CHECKPOINT_TEMPLATE_TAG,
+ SOURCE_STRATEGY_OPTIONS,
+ TARGET_STRATEGY_OPTIONS,
+} from "metabase-enterprise/transforms/constants";
+import type { NewTransformValues } from "metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal";
+import * as Lib from "metabase-lib";
+import Question from "metabase-lib/v1/Question";
+import type { TransformSource } from "metabase-types/api";
+
+export const IncrementalTransformSettings = ({
+ source,
+ checkOnMount,
+}: {
+ source: TransformSource;
+ checkOnMount?: boolean;
+}) => {
+ const metadata = useSelector(getMetadata);
+ const { values } = useFormikContext<NewTransformValues>();
+ // Convert DatasetQuery to Lib.Query via Question
+ const libQuery = useMemo(() => {
+ if (source.type !== "query") {
+ return null;
+ }
+
+ try {
+ const question = Question.create({
+ dataset_query: source.query,
+ metadata,
+ });
+ return question.query();
+ } catch (error) {
+ console.error("SourceStrategyFields: Error creating question", error);
+ return null;
+ }
+ }, [source, metadata]);
+
+ // Check if this is an MBQL query (not native SQL or Python)
+ const isMbqlQuery = useMemo(() => {
+ if (!libQuery) {
+ return false;
+ }
+
+ try {
+ const queryDisplayInfo = Lib.queryDisplayInfo(libQuery);
+ return !queryDisplayInfo.isNative;
+ } catch (error) {
+ console.error("Error checking query type", error);
+ return false;
+ }
+ }, [libQuery]);
+
+ // Check if this is a Python transform with exactly one source table
+ // Incremental transforms are only supported for single-table Python transforms
+ const isPythonTransform = source.type === "python";
+ const isMultiTablePythonTransform =
+ isPythonTransform && Object.keys(source["source-tables"]).length > 1;
+
+ const [checkQueryComplexity, { data: complexity }] =
+ useLazyCheckQueryComplexityQuery();
+ const [showComplexityWarning, setShowComplexityWarning] = useState(false);
+
+ const { transformType, query } = useMemo(() => {
+ if (isMbqlQuery) {
+ return { transformType: "query" as const, query: libQuery };
+ }
+ if (isPythonTransform) {
+ return { transformType: "python" as const, query: null };
+ }
+ return { transformType: "native" as const, query: libQuery };
+ }, [isMbqlQuery, isPythonTransform, libQuery]);
+
+ const hasCheckpointTag = useMemo(() => {
+ if (!libQuery || transformType !== "native") {
+ return false;
+ }
+ const tags = Lib.templateTags(libQuery);
+ return CHECKPOINT_TEMPLATE_TAG in tags;
+ }, [libQuery, transformType]);
+
+ useEffect(() => {
+ async function checkExistingQueryComplexity() {
+ if (
+ checkOnMount &&
+ transformType === "native" &&
+ libQuery &&
+ !hasCheckpointTag &&
+ "source-incremental-strategy" in source
+ ) {
+ const { is_simple } = await checkQueryComplexity(
+ Lib.rawNativeQuery(libQuery),
+ true,
+ ).unwrap();
+ setShowComplexityWarning(!is_simple);
+ }
+ }
+ checkExistingQueryComplexity();
+ }, [
+ checkOnMount,
+ checkQueryComplexity,
+ libQuery,
+ transformType,
+ source,
+ hasCheckpointTag,
+ ]);
+
+ return (
+ <>
+ <FormSwitch
+ disabled={isMultiTablePythonTransform}
+ name="incremental"
+ label={
+ isMultiTablePythonTransform
+ ? t`Incremental transforms are only supported for single data source transforms.`
+ : t`Incremental?`
+ }
+ onChange={async (e) => {
+ if (
+ e.target.checked &&
+ transformType === "native" &&
+ !hasCheckpointTag &&
+ query
+ ) {
+ const complexity = await checkQueryComplexity(
+ Lib.rawNativeQuery(query),
+ true,
+ ).unwrap();
+ setShowComplexityWarning(complexity?.is_simple === false);
+ }
+ }}
+ />
+ {values?.incremental && (
+ <>
+ {showComplexityWarning && (
+ <Alert variant="info" icon="info">
+ <Stack gap="xs">
+ <span>
+ {t`This query is too complex to allow automatic checkpoint column selection. You may need to explicitely add a conditional filter in your query, for example:`}
+ </span>
+ <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>
+ <span>
+ {t`Reason: `}
+ <strong>{complexity?.reason}</strong>
+ </span>
+ </Stack>
+ </Alert>
+ )}
+ <SourceStrategyFields
+ source={source}
+ query={query}
+ type={transformType}
+ />
+ <TargetStrategyFields />
+ </>
+ )}
+ </>
+ );
+};
+
+function TargetStrategyFields() {
+ const { values } = useFormikContext<NewTransformValues>();
+
+ if (!values.incremental) {
+ return null;
+ }
+
+ return (
+ <>
+ {TARGET_STRATEGY_OPTIONS.length > 1 && (
+ <FormSelect
+ name="targetStrategy"
+ label={t`Target Strategy`}
+ description={t`How to update the target table`}
+ data={TARGET_STRATEGY_OPTIONS}
+ />
+ )}
+ {/* Append strategy has no additional fields */}
+ {/* Future strategies like "merge" could add fields here */}
+ </>
+ );
+}
+
+type SourceStrategyFieldsProps = {
+ source: TransformSource;
+ query: Lib.Query | null;
+ type: "query" | "native" | "python";
+};
+
+function SourceStrategyFields({
+ source,
+ query,
+ type,
+}: SourceStrategyFieldsProps) {
+ const { values } = useFormikContext<NewTransformValues>();
+ if (!values.incremental) {
+ return null;
+ }
+
+ return (
+ <>
+ {SOURCE_STRATEGY_OPTIONS.length > 1 && (
+ <FormSelect
+ name="sourceStrategy"
+ label={t`Source Strategy`}
+ description={t`How to track which rows to process`}
+ data={SOURCE_STRATEGY_OPTIONS}
+ />
+ )}
+ {values.sourceStrategy === "checkpoint" && (
+ <>
+ {type === "query" && query && (
+ <KeysetColumnSelect
+ name="checkpointFilterUniqueKey"
+ label={t`Source Filter Field`}
+ placeholder={t`Select a field to filter on`}
+ description={t`Which field from the source to use in the incremental filter`}
+ query={query}
+ />
+ )}
+ {type === "native" && query && (
+ <NativeQueryColumnSelect
+ name="checkpointFilter"
+ label={t`Source Filter Field`}
+ placeholder={t`e.g. id, created_at`}
+ description={t`Column to use in the incremental filter`}
+ query={query}
+ />
+ )}
+ {type === "python" && "source-tables" in source && (
+ <PythonKeysetColumnSelect
+ name="checkpointFilterUniqueKey"
+ label={t`Source Filter Field`}
+ placeholder={t`Select a field to filter on`}
+ description={t`Which field from the source to use in the incremental filter`}
+ sourceTables={source["source-tables"]}
+ />
+ )}
+ </>
+ )}
+ </>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
new file mode 100644
index 00000000000..cccff014020
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
@@ -0,0 +1,88 @@
+import { useMemo } from "react";
+
+import { FormSelect } from "metabase/forms";
+import type { SelectOption } from "metabase/ui";
+import * as Lib from "metabase-lib";
+
+type KeysetColumnSelectProps = {
+ name: string;
+ label: string;
+ placeholder: string;
+ description: string;
+ query: Lib.Query | null;
+ disabled?: boolean;
+};
+
+export function KeysetColumnSelect({
+ name,
+ label,
+ placeholder,
+ description,
+ query,
+ disabled,
+}: KeysetColumnSelectProps) {
+ const columnOptions = useMemo((): Array<SelectOption> => {
+ if (!query) {
+ return [];
+ }
+
+ try {
+ // Use -1 to get the last stage
+ const stageIndex = -1;
+
+ const returnedColumns = Lib.returnedColumns(query, stageIndex);
+ const filterableColumns = Lib.filterableColumns(query, stageIndex);
+
+ const filterableIdentifiers = new Set(
+ filterableColumns.map((col) => {
+ return Lib.columnKey(col);
+ }),
+ );
+
+ const numericFilterableColumns = returnedColumns.filter((column) => {
+ return (
+ filterableIdentifiers.has(Lib.columnKey(column)) &&
+ (Lib.isNumeric(column) || Lib.isTemporal(column))
+ );
+ });
+
+ // Convert to select options with unique keys
+ const seenKeys = new Set<string>();
+ const uniqueColumns: Array<SelectOption> = [];
+
+ numericFilterableColumns.forEach((column) => {
+ const columnInfo = Lib.displayInfo(query, stageIndex, column);
+ const uniqueKey = Lib.columnUniqueKey(column);
+
+ if (!seenKeys.has(uniqueKey)) {
+ seenKeys.add(uniqueKey);
+
+ const label = columnInfo.longDisplayName;
+
+ uniqueColumns.push({ value: uniqueKey, label });
+ }
+ });
+
+ return uniqueColumns;
+ } catch (error) {
+ // If we can't extract columns (e.g., invalid query), return empty array
+ console.error(
+ "KeysetColumnSelect: Error extracting columns from query:",
+ error,
+ );
+ return [];
+ }
+ }, [query]);
+
+ return (
+ <FormSelect
+ name={name}
+ label={label}
+ placeholder={placeholder}
+ description={description}
+ data={columnOptions}
+ searchable
+ disabled={disabled || columnOptions.length === 0}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
new file mode 100644
index 00000000000..db49a96cf31
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
@@ -0,0 +1,71 @@
+import { skipToken } from "@reduxjs/toolkit/query";
+import { useMemo } from "react";
+
+import { useGetTableQueryMetadataQuery } from "metabase/api";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
+import * as Lib from "metabase-lib";
+import type { PythonTransformTableAliases } from "metabase-types/api";
+
+import { KeysetColumnSelect } from "./KeysetColumnSelect";
+
+type PythonKeysetColumnSelectProps = {
+ name: string;
+ label: string;
+ placeholder: string;
+ description: string;
+ sourceTables: PythonTransformTableAliases;
+};
+
+export function PythonKeysetColumnSelect({
+ name,
+ label,
+ placeholder,
+ description,
+ sourceTables,
+}: PythonKeysetColumnSelectProps) {
+ const metadata = useSelector(getMetadata);
+
+ // Get the first (and should be only) table ID
+ // Incremental transforms are only supported for single-table Python transforms
+ const tableId = useMemo(() => {
+ const tableIds = Object.values(sourceTables);
+ return tableIds.length === 1 ? tableIds[0] : null;
+ }, [sourceTables]);
+
+ // Fetch metadata for the table
+ const {
+ data: table,
+ isLoading,
+ error,
+ } = useGetTableQueryMetadataQuery(tableId ? { id: tableId } : skipToken);
+
+ // Create a query from the table to get column metadata
+ const query = useMemo(() => {
+ if (!table || !table.db_id) {
+ return null;
+ }
+
+ try {
+ const metadataProvider = Lib.metadataProvider(table.db_id, metadata);
+ const tableMetadata = Lib.tableOrCardMetadata(metadataProvider, table.id);
+ if (!tableMetadata) {
+ return null;
+ }
+ return Lib.queryFromTableOrCardMetadata(metadataProvider, tableMetadata);
+ } catch {
+ return null;
+ }
+ }, [table, metadata]);
+
+ return (
+ <KeysetColumnSelect
+ name={name}
+ label={label}
+ placeholder={placeholder}
+ description={description}
+ query={query}
+ disabled={isLoading || !!error}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
new file mode 100644
index 00000000000..fda664b4009
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
@@ -0,0 +1,2 @@
+export { KeysetColumnSelect } from "./KeysetColumnSelect";
+export { PythonKeysetColumnSelect } from "./PythonKeysetColumnSelect";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx
new file mode 100644
index 00000000000..6b63e154dd8
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx
@@ -0,0 +1,90 @@
+import { useEffect, useState } from "react";
+import { useLatest } from "react-use";
+
+import { FormSelect, FormTextInput } from "metabase/forms";
+import { Loader } from "metabase/ui";
+import { useExtractColumnsFromQueryMutation } from "metabase-enterprise/api";
+import * as Lib from "metabase-lib";
+
+type NativeQueryColumnSelectProps = {
+ name: string;
+ label: string;
+ description: string;
+ placeholder: string;
+ query: Lib.Query;
+};
+
+export function NativeQueryColumnSelect({
+ name,
+ label,
+ description,
+ placeholder,
+ query,
+}: NativeQueryColumnSelectProps) {
+ const [columns, setColumns] = useState<string[] | null>(null);
+ const [extractColumns, { isLoading }] = useExtractColumnsFromQueryMutation();
+
+ const queryRef = useLatest(query);
+
+ useEffect(() => {
+ let isCancelled = false;
+ if (
+ columns &&
+ Lib.areLegacyQueriesEqual(
+ Lib.toLegacyQuery(queryRef.current),
+ Lib.toLegacyQuery(query),
+ )
+ ) {
+ return;
+ }
+
+ const extract = async () => {
+ try {
+ const result = await extractColumns({
+ query: Lib.toJsQuery(query),
+ }).unwrap();
+
+ if (!isCancelled) {
+ // Only use columns if we got at least one
+ setColumns(result.columns.length > 0 ? result.columns : null);
+ }
+ } catch (error) {
+ // On error, fall back to text input
+ if (!isCancelled) {
+ setColumns(null);
+ }
+ }
+ };
+
+ extract();
+
+ return () => {
+ isCancelled = true;
+ };
+ }, [extractColumns, query, queryRef, columns]);
+
+ // If we successfully extracted columns, show a selector
+ if (columns && columns.length > 0) {
+ return (
+ <FormSelect
+ name={name}
+ rightSection={isLoading ? <Loader size="sm" /> : null}
+ disabled={isLoading}
+ label={label}
+ description={description}
+ placeholder={placeholder}
+ data={columns.map((col) => ({ value: col, label: col }))}
+ />
+ );
+ }
+
+ // Otherwise, fall back to text input
+ return (
+ <FormTextInput
+ name={name}
+ label={label}
+ placeholder={placeholder}
+ description={description}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts
new file mode 100644
index 00000000000..b95d8e9d607
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts
@@ -0,0 +1 @@
+export { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
index 809e313f487..94ed1f7f280 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
@@ -47,7 +47,13 @@ export function TransformEditor({
const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);
const handleQueryChange = (query: Lib.Query) => {
- onChangeSource({ type: "query", query: Lib.toJsQuery(query) });
+ const newSource: QueryTransformSource = {
+ ...source,
+ type: "query",
+ query: Lib.toJsQuery(query),
+ };
+
+ onChangeSource(newSource);
};
return (
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts
index bbe97338858..18107133265 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts
@@ -1,3 +1,5 @@
+import { t } from "ttag";
+
import { isCypressActive } from "metabase/env";
export const NAME_MAX_LENGTH = 254;
@@ -6,3 +8,22 @@ export const POLLING_INTERVAL = isCypressActive ? 200 : 3000;
export const FILTER_WIDGET_MIN_WIDTH = 300;
export const FILTER_WIDGET_MAX_HEIGHT = 400;
+
+export const CHECKPOINT_TEMPLATE_TAG = "checkpoint";
+export const SOURCE_STRATEGY_OPTIONS = [
+ {
+ value: CHECKPOINT_TEMPLATE_TAG,
+ get label() {
+ return t`Checkpoint`;
+ },
+ },
+];
+
+export const TARGET_STRATEGY_OPTIONS = [
+ {
+ value: "append" as const,
+ get label() {
+ return t`Append`;
+ },
+ },
+];
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
index 4a18760e67d..68cee1087b4 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
@@ -8,7 +8,9 @@ import {
useGetDatabaseQuery,
useListDatabaseSchemasQuery,
} from "metabase/api";
+import { getErrorMessage } from "metabase/api/utils";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useToast } from "metabase/common/hooks";
import {
Form,
FormErrorMessage,
@@ -19,6 +21,7 @@ import {
import * as Errors from "metabase/lib/errors";
import { Box, Button, Group, Modal, Stack } from "metabase/ui";
import { useCreateTransformMutation } from "metabase-enterprise/api";
+import { IncrementalTransformSettings } from "metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings";
import type {
CreateTransformRequest,
Transform,
@@ -26,15 +29,27 @@ import type {
} from "metabase-types/api";
import { trackTransformCreated } from "../../../analytics";
-import type { NewTransformValues } from "../types";
import { SchemaFormSelect } from "./../../../components/SchemaFormSelect";
-const NEW_TRANSFORM_SCHEMA = Yup.object({
- name: Yup.string().required(Errors.required),
- targetName: Yup.string().required(Errors.required),
- targetSchema: Yup.string().nullable(),
-});
+function getValidationSchema() {
+ return Yup.object({
+ name: Yup.string().required(Errors.required),
+ targetName: Yup.string().required(Errors.required),
+ targetSchema: Yup.string().nullable().defined(),
+ incremental: Yup.boolean().required(),
+ // For native queries, use checkpointFilter (plain string)
+ checkpointFilter: Yup.string().nullable(),
+ // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
+ checkpointFilterUniqueKey: Yup.string().nullable(),
+ sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
+ targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
+ });
+}
+
+export type NewTransformValues = Yup.InferType<
+ ReturnType<typeof getValidationSchema>
+>;
type CreateTransformModalProps = {
source: TransformSource;
@@ -74,6 +89,7 @@ function CreateTransformForm({
onCreate,
onClose,
}: CreateTransformFormProps) {
+ const [sendToast] = useToast();
const databaseId =
source.type === "query" ? source.query.database : source["source-database"];
@@ -102,6 +118,8 @@ function CreateTransformForm({
[schemas, defaultValues],
);
+ const validationSchema = useMemo(() => getValidationSchema(), []);
+
if (isLoading || error != null) {
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
}
@@ -111,21 +129,26 @@ function CreateTransformForm({
throw new Error("Database ID is required");
}
const request = getCreateRequest(source, values, databaseId);
- const transform = await createTransform(request).unwrap();
-
- trackTransformCreated({ transformId: transform.id });
-
- onCreate(transform);
+ try {
+ const transform = await createTransform(request).unwrap();
+ trackTransformCreated({ transformId: transform.id });
+ onCreate(transform);
+ } catch (error) {
+ sendToast({
+ message: getErrorMessage(error, t`Failed to create transform`),
+ icon: "warning",
+ });
+ }
};
return (
<FormProvider
initialValues={initialValues}
- validationSchema={NEW_TRANSFORM_SCHEMA}
+ validationSchema={validationSchema}
onSubmit={handleSubmit}
>
<Form>
- <Stack gap="lg">
+ <Stack gap="lg" mt="sm">
<FormTextInput
name="name"
label={t`Name`}
@@ -144,6 +167,7 @@ function CreateTransformForm({
label={t`Table name`}
placeholder={t`descriptive_name`}
/>
+ <IncrementalTransformSettings source={source} />
<Group>
<Box flex={1}>
<FormErrorMessage />
@@ -166,22 +190,71 @@ function getInitialValues(
targetName: "",
targetSchema: schemas?.[0] || null,
...defaultValues,
+ checkpointFilter: null,
+ checkpointFilterUniqueKey: null,
+ incremental: false,
+ sourceStrategy: "checkpoint",
+ targetStrategy: "append",
};
}
function getCreateRequest(
source: TransformSource,
- { name, targetName, targetSchema }: NewTransformValues,
+ {
+ name,
+ targetName,
+ targetSchema,
+ incremental,
+ checkpointFilter,
+ checkpointFilterUniqueKey,
+ sourceStrategy,
+ targetStrategy,
+ }: NewTransformValues,
databaseId: number,
): CreateTransformRequest {
+ // Build the source with incremental strategy if enabled
+ let transformSource: TransformSource;
+ if (incremental) {
+ // For native queries, use checkpoint-filter (plain string)
+ // For MBQL/Python queries, use checkpoint-filter-unique-key (prefixed format)
+ const strategyFields = checkpointFilter
+ ? { "checkpoint-filter": checkpointFilter }
+ : checkpointFilterUniqueKey
+ ? { "checkpoint-filter-unique-key": checkpointFilterUniqueKey }
+ : {};
+
+ transformSource = {
+ ...source,
+ "source-incremental-strategy": {
+ type: sourceStrategy,
+ ...strategyFields,
+ },
+ };
+ } else {
+ transformSource = source;
+ }
+
+ // Build the target with incremental strategy if enabled
+ const transformTarget: CreateTransformRequest["target"] = incremental
+ ? {
+ type: "table-incremental",
+ name: targetName,
+ schema: targetSchema,
+ database: databaseId,
+ "target-incremental-strategy": {
+ type: targetStrategy,
+ },
+ }
+ : {
+ type: "table",
+ name: targetName,
+ schema: targetSchema ?? null,
+ database: databaseId,
+ };
+
return {
- name: name,
- source,
- target: {
- type: "table",
- name: targetName,
- schema: targetSchema ?? null,
- database: databaseId,
- },
+ name,
+ source: transformSource,
+ target: transformTarget,
};
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts
index b8a1ea8b56b..e69de29bb2d 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts
@@ -1,5 +0,0 @@
-export type NewTransformValues = {
- name: string;
- targetName: string;
- targetSchema: string | null;
-};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts
index c537e3464b0..3343ca5e8eb 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts
@@ -6,7 +6,7 @@ import type {
SuggestedTransform,
} from "metabase-types/api";
-import type { NewTransformValues } from "./types";
+import type { NewTransformValues } from "./CreateTransformModal/CreateTransformModal";
export function getInitialQuerySource(): QueryTransformSource {
const question = Question.create({ DEPRECATED_RAW_MBQL_type: "query" });
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx
index b6f82bd14bb..1469687aa4e 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx
@@ -24,6 +24,7 @@ import type { Transform } from "metabase-types/api";
import { TitleSection } from "../../../components/TitleSection";
import { isTransformRunning, sourceDatabaseId } from "../../../utils";
+import { UpdateIncrementalModal } from "./UpdateIncrementalModal";
import { UpdateTargetModal } from "./UpdateTargetModal";
type TargetSectionProps = {
@@ -42,6 +43,7 @@ export function TargetSection({ transform }: TargetSectionProps) {
<Divider />
<Group p="lg">
<EditTargetButton transform={transform} />
+ <EditIncrementalButton transform={transform} />
<EditMetadataButton transform={transform} />
</Group>
</TitleSection>
@@ -196,6 +198,44 @@ function EditTargetButton({ transform }: EditTargetButtonProps) {
);
}
+type EditIncrementalButtonProps = {
+ transform: Transform;
+};
+
+function EditIncrementalButton({ transform }: EditIncrementalButtonProps) {
+ const [isModalOpened, { open: openModal, close: closeModal }] =
+ useDisclosure();
+ const { sendSuccessToast } = useMetadataToasts();
+
+ const isIncremental = transform.target.type === "table-incremental";
+
+ const handleUpdate = () => {
+ closeModal();
+ sendSuccessToast(t`Incremental settings updated`);
+ };
+
+ return (
+ <>
+ <Button
+ leftSection={<Icon name="refresh" aria-hidden />}
+ disabled={isTransformRunning(transform)}
+ onClick={openModal}
+ >
+ {isIncremental
+ ? t`Edit incremental settings`
+ : t`Make transform incremental`}
+ </Button>
+ {isModalOpened && (
+ <UpdateIncrementalModal
+ transform={transform}
+ onUpdate={handleUpdate}
+ onClose={closeModal}
+ />
+ )}
+ </>
+ );
+}
+
type EditMetadataButtonProps = {
transform: Transform;
};
@@ -213,7 +253,7 @@ function EditMetadataButton({ transform }: EditMetadataButtonProps) {
leftSection={<Icon name="label" aria-hidden />}
data-testid="table-metadata-link"
>
- {t`Edit this table’s metadata`}
+ {t`Edit this table's metadata`}
</Button>
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsx
new file mode 100644
index 00000000000..92205c5f069
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsx
@@ -0,0 +1,189 @@
+import { useMemo } from "react";
+import { t } from "ttag";
+import * as Yup from "yup";
+
+import { getErrorMessage } from "metabase/api/utils";
+import { useToast } from "metabase/common/hooks";
+import {
+ Form,
+ FormErrorMessage,
+ FormProvider,
+ FormSubmitButton,
+} from "metabase/forms";
+import { Box, Button, FocusTrap, Group, Modal, Stack } from "metabase/ui";
+import { useUpdateTransformMutation } from "metabase-enterprise/api";
+import { IncrementalTransformSettings } from "metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings";
+import type { Transform } from "metabase-types/api";
+
+type UpdateIncrementalModalProps = {
+ transform: Transform;
+ onUpdate: () => void;
+ onClose: () => void;
+};
+
+export function UpdateIncrementalModal({
+ transform,
+ onUpdate,
+ onClose,
+}: UpdateIncrementalModalProps) {
+ return (
+ <Modal
+ title={t`Configure incremental settings`}
+ opened
+ padding="xl"
+ onClose={onClose}
+ >
+ <FocusTrap.InitialFocus />
+ <UpdateIncrementalForm
+ transform={transform}
+ onUpdate={onUpdate}
+ onClose={onClose}
+ />
+ </Modal>
+ );
+}
+
+type IncrementalValues = {
+ incremental: boolean;
+ sourceStrategy: "checkpoint";
+ checkpointFilter: string | null;
+ checkpointFilterUniqueKey: string | null;
+ targetStrategy: "append";
+};
+
+function getValidationSchema() {
+ return Yup.object({
+ incremental: Yup.boolean().required(),
+ sourceStrategy: Yup.string().oneOf(["checkpoint"]).required(),
+ // For native queries, use checkpointFilter (plain string)
+ checkpointFilter: Yup.string().nullable(),
+ // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
+ checkpointFilterUniqueKey: Yup.string().nullable(),
+ targetStrategy: Yup.string().oneOf(["append"]).required(),
+ });
+}
+
+type UpdateIncrementalFormProps = {
+ transform: Transform;
+ onUpdate: () => void;
+ onClose: () => void;
+};
+
+function UpdateIncrementalForm({
+ transform,
+ onUpdate,
+ onClose,
+}: UpdateIncrementalFormProps) {
+ const [sendToast] = useToast();
+ const [updateTransform] = useUpdateTransformMutation();
+ const initialValues = useMemo(() => getInitialValues(transform), [transform]);
+ const validationSchema = useMemo(() => getValidationSchema(), []);
+
+ const handleSubmit = async (values: IncrementalValues) => {
+ // Build the source with incremental strategy if enabled
+ let source;
+ if (values.incremental) {
+ // For native queries, use checkpoint-filter (plain string)
+ // For MBQL/Python queries, use checkpoint-filter-unique-key (prefixed format)
+ const strategyFields = values.checkpointFilter
+ ? { "checkpoint-filter": values.checkpointFilter }
+ : values.checkpointFilterUniqueKey
+ ? { "checkpoint-filter-unique-key": values.checkpointFilterUniqueKey }
+ : {};
+
+ source = {
+ ...transform.source,
+ "source-incremental-strategy": {
+ type: "checkpoint" as const,
+ ...strategyFields,
+ },
+ };
+ } else {
+ source = {
+ ...transform.source,
+ "source-incremental-strategy": undefined,
+ };
+ }
+
+ // Build the target with incremental strategy if enabled
+ const target = values.incremental
+ ? {
+ type: "table-incremental" as const,
+ name: transform.target.name,
+ schema: transform.target.schema,
+ database: transform.target.database,
+ "target-incremental-strategy": {
+ type: values.targetStrategy,
+ },
+ }
+ : {
+ type: "table" as const,
+ name: transform.target.name,
+ schema: transform.target.schema,
+ database: transform.target.database,
+ };
+
+ try {
+ await updateTransform({
+ id: transform.id,
+ source,
+ target,
+ }).unwrap();
+
+ onUpdate();
+ } catch (error) {
+ sendToast({
+ message: getErrorMessage(error, t`Failed to update transform`),
+ icon: "warning",
+ });
+ }
+ };
+
+ return (
+ <FormProvider
+ initialValues={initialValues}
+ validationSchema={validationSchema}
+ onSubmit={handleSubmit}
+ >
+ <Form>
+ <Stack gap="lg" mt="sm">
+ <IncrementalTransformSettings
+ source={transform.source}
+ checkOnMount
+ />
+ <Group>
+ <Box flex={1}>
+ <FormErrorMessage />
+ </Box>
+ <Button variant="subtle" onClick={onClose}>{t`Cancel`}</Button>
+ <FormSubmitButton label={t`Save`} variant="filled" />
+ </Group>
+ </Stack>
+ </Form>
+ </FormProvider>
+ );
+}
+
+function getInitialValues(transform: Transform): IncrementalValues {
+ const isIncremental = transform.target.type === "table-incremental";
+ const strategy = transform.source["source-incremental-strategy"];
+
+ // Read both fields from the strategy, whichever is present
+ const checkpointFilter =
+ strategy?.type === "checkpoint"
+ ? (strategy["checkpoint-filter"] ?? null)
+ : null;
+
+ const checkpointFilterUniqueKey =
+ strategy?.type === "checkpoint"
+ ? (strategy["checkpoint-filter-unique-key"] ?? null)
+ : null;
+
+ return {
+ incremental: isIncremental,
+ sourceStrategy: "checkpoint",
+ checkpointFilter: isIncremental ? checkpointFilter : null,
+ checkpointFilterUniqueKey: isIncremental ? checkpointFilterUniqueKey : null,
+ targetStrategy: "append",
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
index d9e0e657856..e8e381f5ddd 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
@@ -15,6 +15,8 @@ import type {
TransformSource,
} from "metabase-types/api";
+import { CHECKPOINT_TEMPLATE_TAG } from "./constants";
+
export function parseTimestampWithTimezone(
timestamp: string,
systemTimezone: string | undefined,
@@ -174,6 +176,8 @@ export function isNotDraftSource(
return source.type !== "python" || source["source-database"] != null;
}
+const ALLOWED_TRANSFORM_VARIABLES = [CHECKPOINT_TEMPLATE_TAG];
+
export type ValidationResult = {
isValid: boolean;
errorMessage?: string;
@@ -183,7 +187,14 @@ export function getValidationResult(query: Lib.Query): ValidationResult {
const { isNative } = Lib.queryDisplayInfo(query);
if (isNative) {
const tags = Object.values(Lib.templateTags(query));
- if (tags.some((t) => t.type !== "card" && t.type !== "snippet")) {
+ // Allow snippets, cards, and the special transform variables ({checkpoint})
+ const hasInvalidTags = tags.some(
+ (t) =>
+ t.type !== "card" &&
+ t.type !== "snippet" &&
+ !ALLOWED_TRANSFORM_VARIABLES.includes(t.name),
+ );
+ if (hasInvalidTags) {
return {
isValid: false,
errorMessage: t`In transforms, you can use snippets and question or model references, but not variables.`,
diff --git a/frontend/src/metabase-lib/metadata.ts b/frontend/src/metabase-lib/metadata.ts
index 03fdeb642ca..055748fb616 100644
--- a/frontend/src/metabase-lib/metadata.ts
+++ b/frontend/src/metabase-lib/metadata.ts
@@ -230,6 +230,10 @@ export function columnKey(column: ColumnMetadata): string {
return ML.column_key(column);
}
+export function columnUniqueKey(column: ColumnMetadata): string {
+ return ML.column_unique_key(column);
+}
+
export function isColumnMetadata(arg: unknown): arg is ColumnMetadata {
return ML.column_metadata_QMARK_(arg);
}
diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.ts
index c5651f120ed..27312d6fdd1 100644
--- a/frontend/src/metabase-types/api/mocks/transform.ts
+++ b/frontend/src/metabase-types/api/mocks/transform.ts
@@ -39,12 +39,28 @@ export function createMockPythonTransformSource({
export function createMockTransformTarget(
opts?: Partial<TransformTarget>,
): TransformTarget {
- return {
- type: "table",
+ const base = {
+ type: "table" as const,
name: "Table",
schema: null,
database: 1,
+ };
+
+ if (opts?.type === "table-incremental") {
+ return {
+ ...base,
+ ...opts,
+ type: "table-incremental",
+ "target-incremental-strategy": opts["target-incremental-strategy"] ?? {
+ type: "append",
+ },
+ };
+ }
+
+ return {
+ ...base,
...opts,
+ type: "table",
};
}
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index ace4df4ff1f..86a5788f813 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -29,6 +29,16 @@ export type SuggestedTransform = Partial<Pick<Transform, "id">> &
export type PythonTransformTableAliases = Record<string, ConcreteTableId>;
+export type TransformSourceCheckpointStrategy = {
+ type: "checkpoint";
+ // For native queries
+ "checkpoint-filter"?: string;
+ // For MBQL and Python queries
+ "checkpoint-filter-unique-key"?: string;
+};
+
+export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;
+
export type PythonTransformSourceDraft = {
type: "python";
body: string;
@@ -41,15 +51,20 @@ export type PythonTransformSource = {
body: string;
"source-database": DatabaseId;
"source-tables": PythonTransformTableAliases;
+ "source-incremental-strategy"?: SourceIncrementalStrategy;
};
export type QueryTransformSource = {
type: "query";
query: DatasetQuery;
+ "source-incremental-strategy"?: SourceIncrementalStrategy;
};
export type TransformSource = QueryTransformSource | PythonTransformSource;
+export type TransformTargetAppendStrategy = {
+ type: "append";
+};
export type DraftTransformSource =
| Transform["source"]
| PythonTransformSourceDraft;
@@ -58,15 +73,27 @@ export type DraftTransform = Partial<
Pick<Transform, "id" | "name" | "description" | "target">
> & { source: DraftTransformSource };
-export type TransformTargetType = "table";
+export type TargetIncrementalStrategy = TransformTargetAppendStrategy;
+
+export type TransformTargetType = "table" | "table-incremental";
+
+export type TableTarget = {
+ type: "table";
+ name: string;
+ schema: string | null;
+ database: number;
+};
-export type TransformTarget = {
- type: TransformTargetType;
+export type TableIncrementalTarget = {
+ type: "table-incremental";
name: string;
schema: string | null;
database: number;
+ "target-incremental-strategy": TargetIncrementalStrategy;
};
+export type TransformTarget = TableTarget | TableIncrementalTarget;
+
export type TransformRun = {
id: TransformRunId;
status: TransformRunStatus;
@@ -212,3 +239,18 @@ export type UpdatePythonLibraryRequest = {
path: string;
source: string;
};
+
+export type ExtractColumnsFromQueryRequest = {
+ query: DatasetQuery;
+};
+
+export type ExtractColumnsFromQueryResponse = {
+ columns: string[];
+};
+
+export type CheckQueryComplexityRequest = string;
+
+export type CheckQueryComplexityResponse = {
+ is_simple: boolean;
+ reason: string;
+};
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 9e210d7b80d..38c9032920d 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -878,8 +878,17 @@
(defmethod driver/compile-transform :bigquery-cloud-sdk
[_driver {:keys [query output-table]}]
- (let [table-str (get-table-str output-table)]
- [(format "CREATE OR REPLACE TABLE %s AS %s" table-str query)]))
+ (let [{sql-query :query sql-params :params} query
+ table-str (get-table-str output-table)]
+ [(format "CREATE OR REPLACE TABLE %s AS %s" table-str sql-query)
+ sql-params]))
+
+(defmethod driver/compile-insert :bigquery-cloud-sdk
+ [_driver {:keys [query output-table]}]
+ (let [{sql-query :query sql-params :params} query
+ table-str (get-table-str output-table)]
+ [(format "INSERT INTO %s %s" table-str sql-query)
+ sql-params]))
(defmethod driver/compile-drop-table :bigquery-cloud-sdk
[_driver table]
@@ -927,7 +936,7 @@
(u.date/format (u.date/parse value))
:type/DateTime
- (u.date/format (u.date/parse value))
+ (u.date/format :iso-local-date-time (u.date/parse value))
:type/DateTimeWithLocalTZ
(u.date/format (u.date/parse value))
@@ -974,9 +983,10 @@
(try
(doall
(for [query queries]
- (let [sql (if (string? query) query (first query))
+ (let [[sql params] (if (string? query) [query] query)
_ (log/debugf "Executing BigQuery DDL: %s" sql)
job-config (-> (QueryJobConfiguration/newBuilder sql)
+ (bigquery.params/set-parameters! params)
(.setUseLegacySql false)
(.build))
table-result (.query client job-config (into-array BigQuery$JobOption []))]
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index 3e5131efe1a..69217c4ba8b 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -1346,3 +1346,14 @@
:type/JSON [[:raw "JSON"]]
:type/SerializedJSON [[:raw "JSON"]]
:type/Decimal [[:raw "BIGDECIMAL"]])))
+
+(deftest ^:parallel compile-transform-test
+ (mt/test-driver :bigquery-cloud-sdk
+ (testing "compile-transform creates CREATE OR REPLACE TABLE"
+ (is (= ["CREATE OR REPLACE TABLE `PRODUCTS_COPY` AS SELECT * FROM products" nil]
+ (driver/compile-transform :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}
+ :output-table :PRODUCTS_COPY}))))
+ (testing "compile-insert generates INSERT INTO"
+ (is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil]
+ (driver/compile-insert :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}
+ :output-table :PRODUCTS_COPY}))))))
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index 98bd56ab095..9cf62520555 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -320,14 +320,21 @@
(defmethod driver/compile-transform :clickhouse
[driver {:keys [query output-table]}]
- (let [pieces [(sql.qp/format-honeysql driver {:create-table output-table})
+ (let [{sql-query :query sql-params :params} query
+ pieces [(sql.qp/format-honeysql driver {:create-table output-table})
;; TODO(rileythomp, 2025-08-22): Is there a better way to do this?
;; i.e. only do this if we don't have a non-nullable field to use as a primary key?
(sql.qp/format-honeysql driver {:raw "ORDER BY ()"})
["AS"]
- (sql.qp/format-honeysql driver {:raw query})]
- query (str/join " " (map first pieces))]
- (into [query] (mapcat rest) pieces)))
+ [sql-query sql-params]]
+ sql (str/join " " (map first pieces))]
+ (into [sql] (mapcat rest) pieces)))
+
+(defmethod driver/compile-insert :clickhouse
+ [driver {:keys [query output-table]}]
+ (let [{sql-query :query sql-params :params} query]
+ [(first (sql.qp/format-honeysql driver {:insert-into [output-table {:raw sql-query}]}))
+ sql-params]))
(defmethod driver/create-schema-if-needed! :clickhouse
[driver conn-spec schema]
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 7166c2adce0..0e442dbb75c 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -316,10 +316,14 @@
(deftest ^:parallel compile-transform-test
(mt/test-driver :clickhouse
- (testing "compile transform for clickhouse with empty primary key column"
- (is (= ["CREATE TABLE `PRODUCTS_COPY` ORDER BY () AS SELECT * FROM products"]
- (driver/compile-transform :clickhouse {:query "SELECT * FROM products"
- :output-table "PRODUCTS_COPY"}))))))
+ (testing "compile-transform for clickhouse with empty primary key column"
+ (is (= ["CREATE TABLE `PRODUCTS_COPY` ORDER BY () AS SELECT * FROM products" nil]
+ (driver/compile-transform :clickhouse {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "compile-insert generates INSERT INTO"
+ (is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil]
+ (driver/compile-insert :clickhouse {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))))
(deftest ^:parallel clickhouse-db-supports-schemas-test
(doseq [[schemas-supported? details] [[false? {}]
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index bc28c143513..5560e5f56b4 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -1024,11 +1024,18 @@
(defmethod driver/compile-transform :sqlserver
[driver {:keys [query output-table]}]
- (let [^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))
- ^Select parsed-query (macaw/parsed-query query)
+ (let [{sql-query :query sql-params :params} query
+ ^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))
+ ^Select parsed-query (macaw/parsed-query sql-query)
^PlainSelect select-body (.getSelectBody parsed-query)]
(.setIntoTables select-body [(Table. table-name)])
- [(str parsed-query)]))
+ [(str parsed-query) sql-params]))
+
+(defmethod driver/compile-insert :sqlserver
+ [driver {:keys [query output-table]}]
+ (let [{sql-query :query sql-params :params} query
+ ^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))]
+ [(format "INSERT INTO %s %s" table-name sql-query) sql-params]))
(defmethod driver/table-exists? :sqlserver
[driver database {:keys [schema name] :as _table}]
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index e25915fcc8f..715ffe4ca67 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -771,3 +771,14 @@
:type/Text [:text]
:type/Time [:time]
:type/UUID [:uniqueidentifier])))
+
+(deftest ^:parallel compile-transform-test
+ (mt/test-driver :sqlserver
+ (testing "compile-transform creates SELECT INTO"
+ (is (= ["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]
+ (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "compile-insert generates INSERT INTO"
+ (is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil]
+ (driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))))
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 249a4a63863..0d94f5c74e5 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -7,7 +7,7 @@
SQL-based drivers can use the `:sql` driver as a parent, and JDBC-based SQL drivers can use `:sql-jdbc`. Both of
these drivers define additional multimethods that child drivers should implement; see [[metabase.driver.sql]] and
[[metabase.driver.sql-jdbc]] for more details."
- (:refer-clojure :exclude [some mapv])
+ (:refer-clojure :exclude [some mapv empty?])
#_{:clj-kondo/ignore [:metabase/modules]}
(:require
[clojure.java.io :as io]
@@ -23,7 +23,7 @@
[metabase.util.json :as json]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.performance :refer [mapv]]
+ [metabase.util.performance :refer [mapv empty?]]
[potemkin :as p]))
(set! *warn-on-reflection* true)
@@ -1167,11 +1167,17 @@
:hierarchy #'hierarchy)
(defmulti compile-transform
- "Compiles the sql for a transform statement, given an inner sql query and a destination."
+ "Compiles the sql for a transform statement (CREATE TABLE AS), given a compiled inner sql query and a destination."
{:added "0.57.0", :arglists '([driver {:keys [query output-table]}])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
+(defmulti compile-insert
+ "Compiles the sql for an insert statement (INSERT INTO ... SELECT), given a compiled inner sql query and a destination."
+ {:added "0.58.0", :arglists '([driver {:keys [query output-table]}])}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
(defmulti compile-drop-table
"Compiles the sql for a drop table statement for a given table."
{:added "0.57.0", :arglists '([driver table])}
@@ -1211,6 +1217,10 @@
[(dispatch-on-initialized-driver driver) (:type transform-details)])
:hierarchy #'hierarchy)
+(defmethod drop-transform-target! [::driver :table-incremental]
+ [driver database target]
+ ((get-method drop-transform-target! [driver :table]) driver database target))
+
(mr/def ::native-query-deps.table-dep
[:map
{:closed true}
@@ -1453,14 +1463,14 @@
(defmethod insert-from-source! [::driver :jsonl-file]
[driver db-id {:keys [columns] :as table-definition} {:keys [file]}]
(with-open [rdr (io/reader file)]
- (let [lines (line-seq rdr)
+ (let [lines (line-seq rdr)
data-rows (map (fn [line]
(let [m (json/decode line)]
(mapv (fn [column]
(let [raw-val (get m (:name column))]
(insert-col->val driver :jsonl-file column raw-val)))
columns)))
- lines)]
+ (filter (comp not empty?) lines))]
(insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))
(defmulti add-columns!
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 0e4ae807fef..7eacf0d78b9 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -25,6 +25,7 @@
[metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
@@ -966,8 +967,16 @@
(defmethod driver/insert-col->val [:mysql :jsonl-file]
[_driver _ column-def v]
- (if (and (string? v) (isa? (:type column-def) :type/DateTimeWithTZ))
- (t/offset-date-time v)
+ (if (string? v)
+ (cond
+ (isa? (:type column-def) :type/DateTimeWithTZ)
+ (t/offset-date-time v)
+
+ (isa? (:type column-def) :type/DateTime)
+ (u.date/parse v)
+
+ :else
+ v)
v))
(defn- parse-grant
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index c359d708ee8..31e00212173 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -127,6 +127,14 @@
overwrite? (cons (driver/compile-drop-table driver output-table)))]
{:rows-affected (last (driver/execute-raw-queries! driver conn-spec queries))}))
+(defmethod driver/run-transform! [:sql :table-incremental]
+ [driver {:keys [conn-spec database output-table] :as transform-details} _opts]
+ (let [queries (if (driver/table-exists? driver database {:schema (namespace output-table)
+ :name (name output-table)})
+ (driver/compile-insert driver transform-details)
+ (driver/compile-transform driver transform-details))]
+ {:rows-affected (last (driver/execute-raw-queries! driver conn-spec [queries]))}))
+
(defn qualified-name
"Return the name of the target table of a transform as a possibly qualified symbol."
[{schema :schema, table-name :name}]
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index c6d6fb67003..5da66ba79b6 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2067,9 +2067,18 @@
(defmethod driver/compile-transform :sql
[driver {:keys [query output-table]}]
- (format-honeysql driver
- {:create-table-as [(keyword output-table)]
- :raw query}))
+ (let [{sql-query :query sql-params :params} query]
+ [(first (format-honeysql driver
+ {:create-table-as [(keyword output-table)]
+ :raw sql-query}))
+ sql-params]))
+
+(defmethod driver/compile-insert :sql
+ [driver {:keys [query output-table]}]
+ (let [{sql-query :query sql-params :params} query]
+ [(first (format-honeysql driver
+ {:insert-into [(keyword output-table) {:raw sql-query}]}))
+ sql-params]))
(defmethod driver/compile-drop-table :sql
[driver table]
diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljs
index 9a831900c31..e1e02706cea 100644
--- a/src/metabase/lib/js.cljs
+++ b/src/metabase/lib/js.cljs
@@ -72,6 +72,7 @@
[metabase.lib.js.metadata :as js.metadata]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
+ [metabase.lib.metadata.column :as lib.metadata.column]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.normalize :as lib.normalize]
[metabase.lib.order-by :as lib.order-by]
@@ -1494,6 +1495,14 @@
lib.convert/->legacy-MBQL
normalize-legacy-ref))
+(defn ^:export column-unique-key
+ "Given a column metadata from eg. [[fieldable-columns]], return an opaque unique key for it.
+ This key is stable based on the column's alias and can be used to identify columns across operations.
+
+ > **Code health:** Healthy. Prefer this over [[ref]] for identifying columns in UI state."
+ [column]
+ (lib.metadata.column/column-unique-key column))
+
(defn ^:export legacy-ref
"Given a column, metric or segment metadata from eg. [[fieldable-columns]] or [[available-segments]],
return it as a legacy JSON field ref.
diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.clj
index d476c472ea1..bd215cbb072 100644
--- a/test/metabase/driver/sql/transforms_test.clj
+++ b/test/metabase/driver/sql/transforms_test.clj
@@ -11,7 +11,7 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(testing "simple table name"
(let [result (driver/compile-transform driver/*driver*
- {:query "SELECT * FROM products"
+ {:query {:query "SELECT * FROM products"}
:output-table :my_table
:primary-key "id"})]
(testing "returns a vector"
@@ -28,7 +28,7 @@
(testing "schema-qualified table name"
(let [result (driver/compile-transform driver/*driver*
- {:query "SELECT * FROM products"
+ {:query {:query "SELECT * FROM products"}
:output-table :my_schema/my_table
:primary-key "id"})]
(testing "returns a vector"
@@ -82,7 +82,7 @@
(testing "execute-transform! should pass correct format to execute-raw-queries!"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(let [compile-result (driver/compile-transform driver/*driver*
- {:query "SELECT * FROM products"
+ {:query {:query "SELECT * FROM products"}
:output-table :my_table
:primary-key "id"})
drop-result (driver/compile-drop-table driver/*driver* :my_table)]
@@ -126,3 +126,32 @@
;; Drivers might quote these differently, but both parts should be present
(is (or (re-find #"schema.*my_table" (first result))
(re-find #"my_table" (first result)))))))))
+
+(deftest compile-insert-test
+ (testing "compile-insert generates INSERT INTO statements"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "simple table name"
+ (let [result (driver/compile-insert driver/*driver*
+ {:query {:query "SELECT * FROM products"}
+ :output-table :my_table})]
+ (testing "returns a vector"
+ (is (vector? result)))
+ (testing "first element is SQL string"
+ (is (string? (first result))))
+ (testing "generates INSERT INTO statement"
+ (is (re-find #"(?i)INSERT\s+INTO.*my_table" (first result))))
+ (testing "includes SELECT statement"
+ (is (re-find #"(?i)SELECT" (first result))))))
+
+ (testing "schema-qualified table"
+ (let [result (driver/compile-insert driver/*driver*
+ {:query {:query "SELECT * FROM products"}
+ :output-table :my_schema/my_table})]
+ (testing "returns a vector"
+ (is (vector? result)))
+ (testing "generates INSERT INTO statement"
+ (is (re-find #"(?i)INSERT\s+INTO" (first result))))
+ (testing "includes both schema and table parts"
+ (let [sql (first result)]
+ (is (re-find #"my_schema" sql) "Schema name should be present")
+ (is (re-find #"my_table" sql) "Table name should be present"))))))))