The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Incremental transforms (#65842)
Stas Gavrylova07c3b654e5d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 0d70159f6ec..5593084cc0d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -680,6 +680,7 @@metabase.lib.join.utilmetabase.lib.metadatametabase.lib.metadata.cached-provider+ metabase.lib.metadata.columnmetabase.lib.metadata.invocation-trackermetabase.lib.metadata.protocolsmetabase.lib.metadata.result-metadata@@ -2276,6 +2277,8 @@app-dbconfigdriver+ lib+ lib-bequery-processorsettingsutildiff --git a/deps.edn b/deps.ednindex e0f6f6aa4d8..f8330dffce1 100644--- a/deps.edn+++ b/deps.edn@@ -860,6 +860,13 @@{:exec-args {:only ["test/metabase/notification""test/metabase/channel"]}}+ ;; test the transforms feature+ ;; clj -X:dev:ee:ee-dev:test:test/transforms+ :test/transforms+ {:exec-args {:only ["enterprise/backend/test/metabase_enterprise/transforms"+ "enterprise/backend/test/metabase_enterprise/transforms_python"]}}++;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP;;;; clj -X:dev:ee:ee-dev:test:test/mbqldiff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex f80d8ba945e..bd866e413c5 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -8,6 +8,8 @@ title: Driver interface changelog- Added a `:collate` feature for drivers that support collation settings on text fields+- Added `metabase.driver/compile-insert` to implement incremental transforms.+## Metabase 0.57.0- `driver/field-reference-mlv2` is now deprecated, and is no longer used. Please remove your implementations.diff --git a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms.cy.spec.tsindex faf742701b5..84dd0a0a34f 100644--- a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms.cy.spec.ts@@ -420,10 +420,10 @@ LIMITcy.findByLabelText("Table name").type(SOURCE_TABLE);cy.button("Save").click();cy.wait("@createTransform");- cy.findByText("A table with that name already exists.").should(- "be.visible",- );});+ H.undoToast()+ .findByText("A table with that name already exists.")+ .should("be.visible");});it("should be able to create a new schema when saving a transform", () => {@@ -1012,14 +1012,14 @@ LIMITcreateMbqlTransform({ visitTransform: true });H.DataStudio.Transforms.targetTab().click();getTransformsContent()- .findByText("Edit this table’s metadata")+ .findByText("Edit this table's metadata").should("not.exist");cy.log("after table creation");H.DataStudio.Transforms.runTab().click();runTransformAndWaitForSuccess();H.DataStudio.Transforms.targetTab().click();- getTransformsContent().findByText("Edit this table’s metadata").click();+ getTransformsContent().findByText("Edit this table's metadata").click();H.DataModel.TableSection.clickField("Name");H.DataModel.FieldSection.getNameInput().clear().type("New name").blur();cy.wait("@updateField");diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex 7c49e9e4912..11f8dbddf54 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.transforms.api(:require+ [macaw.core :as macaw][metabase-enterprise.transforms.api.transform-job][metabase-enterprise.transforms.api.transform-tag][metabase-enterprise.transforms.canceling :as transforms.canceling]@@ -8,47 +9,39 @@[metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation][metabase-enterprise.transforms.ordering :as transforms.ordering]+ [metabase-enterprise.transforms.schema :as transforms.schema][metabase-enterprise.transforms.util :as transforms.util][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers]+ [metabase.driver :as driver]+ [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u][metabase.events.core :as events]- [metabase.queries.schema :as queries.schema]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.schema :as qp.schema][metabase.request.core :as request][metabase.util :as u][metabase.util.i18n :refer [deferred-tru]][metabase.util.jvm :as u.jvm]+ [metabase.util.log :as log][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][ring.util.response :as response]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.sql PreparedStatement)+ (net.sf.jsqlparser.statement.select PlainSelect)))(comment metabase-enterprise.transforms.api.transform-job/keep-memetabase-enterprise.transforms.api.transform-tag/keep-me)(set! *warn-on-reflection* true)-(mr/def ::transform-source- [:multi {:dispatch (comp keyword :type)}- [:query- [:map- [:type [:= "query"]]- [:query ::queries.schema/query]]]- [:python- [:map {:closed true}- [:source-database {:optional true} :int]- [:source-tables [:map-of :string :int]]- [:type [:= "python"]]- [:body :string]]]])--(mr/def ::transform-target- [:map- [:database {:optional true} :int]- [:type [:enum "table"]]- [:schema {:optional true} [:or ms/NonBlankString :nil]]- [:name :string]])+(mr/def ::transform-source ::transforms.schema/transform-source)++(mr/def ::transform-target ::transforms.schema/transform-target)(mr/def ::run-trigger[:enum "none" "global-schedule"])@@ -70,13 +63,13 @@[transform](let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))(deferred-tru "The target database cannot be found."))- feature (transforms.util/required-database-feature transform)]+ features (transforms.util/required-database-features transform)](api/check-400 (not (:is_sample database))(deferred-tru "Cannot run transforms on the sample database."))(api/check-400 (not (:is_audit database))(deferred-tru "Cannot run transforms on audit databases."))- (api/check-400 (driver.u/supports? (:engine database) feature database)- (deferred-tru "The database does not support the requested transform target type."))+ (api/check-400 (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)+ (deferred-tru "The database does not support the requested transform features."))(api/check-400 (not (transforms.util/db-routing-enabled? database))(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))@@ -117,13 +110,14 @@body :- [:map[:name :string][:description {:optional true} [:maybe :string]]- [:source ::transform-source]- [:target ::transform-target]+ [:source ::transforms.schema/transform-source]+ [:target ::transforms.schema/transform-target][:run_trigger {:optional true} ::run-trigger][:tag_ids {:optional true} [:sequential ms/PositiveInt]]]](api/check-superuser)(check-database-feature body)(check-feature-enabled! body)+(api/check (not (transforms.util/target-table-exists? body))403(deferred-tru "A table with that name already exists."))@@ -200,8 +194,8 @@body :- [:map[:name {:optional true} :string][:description {:optional true} [:maybe :string]]- [:source {:optional true} ::transform-source]- [:target {:optional true} ::transform-target]+ [:source {:optional true} ::transforms.schema/transform-source]+ [:target {:optional true} ::transforms.schema/transform-target][:run_trigger {:optional true} ::run-trigger][:tag_ids {:optional true} [:sequential ms/PositiveInt]]]](api/check-superuser)@@ -281,6 +275,86 @@:run_id run-id})(assoc :status 202)))))+(defn- extract-columns-from-query+ "Attempts to extract column names from an MBQL query without executing it.++ Returns a vector of column names (as strings), or nil if extraction fails.++ The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (try+ (let [{:keys [query]} (qp.compile/compile query)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database-id+ {}+ (fn [conn]+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]+ (when-let [rsmeta (.getMetaData stmt)]+ (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]+ (seq (mapv :name columns))))))))+ (catch Exception e+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))+ nil)))++(defn- simple-native-query?+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion."+ [sql-string]+ (try+ (let [^PlainSelect parsed (macaw/parsed-query sql-string)]+ (cond+ (not (instance? PlainSelect parsed))+ {:is_simple false+ :reason "Not a simple SELECT"}++ (.getLimit parsed)+ {:is_simple false+ :reason "Contains a LIMIT"}++ (.getOffset ^PlainSelect parsed)+ {:is_simple false+ :reason "Contains an OFFSET"}++ (seq (.getWithItemsList ^PlainSelect parsed))+ {:is_simple false+ :reason "Contains a CTE"}++ :else+ {:is_simple true}))+ (catch Exception e+ (log/debugf e "Failed to parse query: %s" (ex-message e))+ {:is_simple false})))++(api.macros/defendpoint :post "/is-simple-query"+ "Checks if a native SQL query string is simple enough for automatic checkpoint insertion"+ [_route-params+ _query-params+ {:keys [query]} :- [:map [:query string?]]]+ (api/check-superuser)+ (simple-native-query? query))++(api.macros/defendpoint :post "/extract-columns"+ "Extract column names from an MBQL query without executing it.++ Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],+ which handles parameterized queries with template tags. Then extracts column names+ using PreparedStatement metadata.++ Returns a map with a :columns key containing a vector of column names (strings).+ If extraction fails, returns nil for :columns."+ [_route-params+ _query-params+ {:keys [query]} :- [:map+ [:query ::qp.schema/any-query]]]+ (api/check-superuser)+ (let [database-id (:database query)+ database (api/check-404 (t2/select-one :model/Database :id database-id))+ driver-name (driver/the-initialized-driver (:engine database))+ columns (extract-columns-from-query driver-name database-id query)]+ {:columns columns}))+(def ^{:arglists '([request respond raise])} routes"`/api/ee/transform` routes."(handlers/routesdiff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex 7f4206eee54..e05ac25b569 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -6,6 +6,7 @@[metabase.driver.util :as driver.u][metabase.events.core :as events][metabase.lib.schema.common :as schema.common]+ [metabase.query-processor.compile :as qp.compile][metabase.util.log :as log][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -14,15 +15,22 @@(mr/def ::transform-details[:map- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]][:conn-spec :any]- [:query :string]+ [:query ::qp.compile/compiled][:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])(mr/def ::transform-opts[:map[:overwrite? :boolean]])+(defn- transform-opts [{:keys [transform-type]}]+ (case transform-type+ :table {:overwrite? true}++ ;; once we have more than just append, dispatch on :target-incremental-strategy+ :table-incremental {}))+(defn run-mbql-transform!"Run `transform` and sync its target table.@@ -34,21 +42,23 @@(try(let [db (get-in source [:query :database]){driver :engine :as database} (t2/select-one :model/Database db)- feature (transforms.util/required-database-feature transform)transform-details {:db-id db+ :database database:transform-id id:transform-type (keyword (:type target)):conn-spec (driver/connection-spec driver database)- :query (transforms.util/compile-source source)+ :query (transforms.util/compile-source transform):output-schema (:schema target):output-table (transforms.util/qualified-table-name driver target)}- opts {:overwrite? true}]+ opts (transform-opts transform-details)+ features (transforms.util/required-database-features transform)]+(when (transforms.util/db-routing-enabled? database)(throw (ex-info "Transforms are not supported on databases with DB routing enabled."{:driver driver, :database database})))- (when-not (driver.u/supports? driver feature database)+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)(throw (ex-info "The database does not support the requested transform target type."- {:driver driver, :database database, :feature feature})))+ {:driver driver, :database database, :features features})));; mark the execution as started and notify any observers(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)](when start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.cljindex 19391a770bc..5088b912630 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj@@ -34,7 +34,8 @@mi/json-out-without-keywordization(update-keys keyword)(m/update-existing :query lib-be/normalize-query)- (m/update-existing :type keyword)))+ (m/update-existing :type keyword)+ (m/update-existing :source-incremental-strategy #(update-keys % keyword))))(defn- transform-source-in [m](-> mdiff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.cljindex cc18d98e565..900256f3194 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj@@ -18,8 +18,8 @@(set! *warn-on-reflection* true)(defmethod transforms.i/table-dependencies :query- [transform]- (let [query (-> (get-in transform [:source :query])+ [{:keys [source]}]+ (let [query (-> (:query source)transforms.util/massage-sql-queryqp.preprocess/preprocess)driver (-> querydiff --git a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms/schema.cljnew file mode 100644index 00000000000..272d6485cc1--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj@@ -0,0 +1,62 @@+(ns metabase-enterprise.transforms.schema+ (:require+ [metabase.lib.metadata.column :as lib.metadata.column]+ [metabase.queries.schema :as queries.schema]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]))++(mr/def ::checkpoint-strategy+ [:map+ [:type [:= "checkpoint"]]+ ;; for native+ [:checkpoint-filter {:optional true} :string]+ ;; for mbql and python+ [:checkpoint-filter-unique-key {:optional true}+ ::lib.metadata.column/column-unique-key]])++(mr/def ::source-incremental-strategy+ [:multi {:dispatch :type}+ ["checkpoint" ::checkpoint-strategy]])++(mr/def ::transform-source+ [:multi {:dispatch (comp keyword :type)}+ [:query+ [:map+ [:type [:= "query"]]+ [:query ::queries.schema/query]+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]+ [:python+ [:map+ [:source-database {:optional true} :int]+ ;; NB: if source is checkpoint, only one table allowed+ [:source-tables [:map-of :string :int]]+ [:type [:= "python"]]+ [:body :string]+ [:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])++(mr/def ::append-config+ [:map [:type [:= "append"]]])++(mr/def ::target-incremental-strategy+ [:multi {:dispatch :type}+ ["append" ::append-config]])++(mr/def ::table-target+ [:map+ [:database {:optional true} :int]+ [:type [:= "table"]]+ [:schema {:optional true} [:or ms/NonBlankString :nil]]+ [:name :string]])++(mr/def ::table-incremental-target+ [:map+ [:database {:optional true} :int]+ [:type [:= "table-incremental"]]+ [:schema {:optional true} [:or ms/NonBlankString :nil]]+ [:name :string]+ [:target-incremental-strategy ::target-incremental-strategy]])++(mr/def ::transform-target+ [:multi {:dispatch :type}+ ["table" ::table-target]+ ["table-incremental" ::table-incremental-target]])diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 2e17951c7c8..f4f57043132 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -8,16 +8,21 @@[metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver]+ [metabase.driver.sql.query-processor :as sql.qp][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.query :as lib.query][metabase.lib.schema.common :as lib.schema.common][metabase.premium-features.core :as premium-features :refer [defenterprise]]+ [metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.parameters.dates :as params.dates][metabase.query-processor.pipeline :as qp.pipeline][metabase.sync.core :as sync][metabase.util :as u][metabase.util.date-2 :as u.date]+ [metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -124,7 +129,7 @@(finally(canceling/chan-end-run! run-id))))-(declare activate-table-and-mark-computed!)+(declare activate-table-and-mark-computed! target-table)(defn sync-target!"Sync target of a transform"@@ -198,19 +203,132 @@[query](assoc-in query [:middleware :disable-remaps?] true))-(defn compile-source- "Compile the source query of a transform."- [{query-type :type :as source}]- (case (keyword query-type)- :query (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))+(defn- checkpoint-incremental?+ "Returns true if `source` uses checkpoint-based incremental strategy."+ [source]+ (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))++(defn- source->checkpoint-filter-unique-key+ "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."+ [query source-incremental-strategy]+ (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))++(defn- source->checkpoint-filter-column+ "Resolve the checkpoint filter column for an incremental transform.++ Tries to resolve the column using the unique key first.+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."+ [query source-incremental-strategy table metadata-provider]+ (or+ (source->checkpoint-filter-unique-key query source-incremental-strategy)+ (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]+ (when-let [field-id (t2/select-one-pk :model/Field+ :table_id (:id table)+ :name field-name)]+ (lib.metadata/field metadata-provider field-id)))))++(defn next-checkpoint+ "Build a query to compute the MAX of the checkpoint column from the target table.++ Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),+ or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."+ [transform-id]+ (let [{:keys [source target] :as transform} (t2/select-one :model/Transform transform-id)+ db-id (transforms.i/target-db-id transform)]+ (when (checkpoint-incremental? source)+ (when-let [table (target-table db-id target)]+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table metadata-provider (:id table))+ query (lib/query metadata-provider table-metadata)]+ (when-let [filter-column (source->checkpoint-filter-column query+ (:source-incremental-strategy source)+ table metadata-provider)]+ {:query (-> query (lib/aggregate (lib/max filter-column)))+ :filter-column filter-column}))))))++(defn- next-checkpoint-value+ "Execute the checkpoint query and normalize the result for database insertion.+ Returns `nil` if the target table is empty."+ [{:keys [query filter-column]}]+ (let [{:keys [base-type]} filter-column+ v (some-> query qp/process-query :data :rows first first)]+ ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right+ ;; shape for reinsertion+ (cond+ (nil? v)+ nil++ (isa? base-type :type/Integer)+ (bigint v)++ ;; any other number that's not an integer, should be a decimal/float+ (number? v)+ (bigdec v)++ :else v)))++(defn preprocess-incremental-query+ "Add checkpoint checkpoint filtering to a query for incremental execution.++ For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.+ For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.+ Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."+ [query source-incremental-strategy checkpoint]+ (if-let [checkpoint-value (next-checkpoint-value checkpoint)]+ (if (lib.query/native? query)+ ;; native query with explicit checkpoint filter+ (if (get-in query [:stages 0 :template-tags "checkpoint"])+ (update query :parameters conj+ {:type (if (number? checkpoint-value) :number :text)+ :target [:variable [:template-tag "checkpoint"]]+ :value checkpoint-value})+ query)+ ;; mbql query+ (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))+ query))++(defn- post-process-incremental-query+ "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.++ Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "+ [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]+ (let [{:keys [checkpoint-filter]} source-incremental-strategy]+ (if (and (lib.query/native? (:query source))+ (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))+ (next-checkpoint-value checkpoint))+ (let [wrap-query (fn [query]+ (let [honeysql-query {:select [:*]+ :from [[[:raw (str "(" query ")")] :subquery]]+ :where [:> (h2x/identifier :field checkpoint-filter)+ [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]+ (first (sql.qp/format-honeysql driver honeysql-query))))]+ (update outer-query :query wrap-query))+ outer-query)))-(defn required-database-feature- "Returns the database feature necessary to execute `transform`."+(defn compile-source+ "Compile the source query of a transform to SQL, applying incremental filtering if required."+ [{:keys [id source]}]+ (let [{:keys [source-incremental-strategy] query-type :type} source]+ (case (keyword query-type)+ :query+ (let [checkpoint (next-checkpoint id)+ query (:query source)+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]+ (binding [driver/*compile-with-inline-parameters*+ (or (= :clickhouse driver)+ driver/*compile-with-inline-parameters*)]+ (-> query+ (preprocess-incremental-query source-incremental-strategy checkpoint)+ massage-sql-query+ qp.compile/compile+ (post-process-incremental-query driver source checkpoint)))))))++(defn required-database-features+ "Returns the database features necessary to execute `transform`."[transform](if (python-transform? transform)- :transforms/python- (case (-> transform :target :type)- "table" :transforms/table)))+ [:transforms/python]+ [:transforms/table]))(defn ->instant"Convert a temporal value `t` to an Instant in the system timezone."diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/api.cljindex 79a807c3a2c..c41c7552c9a 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj@@ -64,7 +64,7 @@(with-open [shared-storage-ref (s3/open-shared-storage! source_tables)](let [server-url (transforms-python.settings/python-runner-url)_ (python-runner/copy-tables-to-s3! {:shared-storage @shared-storage-ref- :table-name->id source_tables+ :source {:source-tables source_tables}:limit per_input_row_limit}){runner-status :statusrunner-body :body}diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 75a6338559f..cabcd70d5da 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -140,22 +140,28 @@ret)))(thunk)))-(defn- create-table-and-insert-data!- "Create a table from metadata and insert data from source."- [driver db-id table-name metadata data-source]- (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))- :columns (mapv (fn [{:keys [name base_type #_database_type]}]- {:name name- :type (python-runner/restricted-insert-type base_type)- ;; :database-type database_type- :nullable? true})- (:fields metadata))}- data-source (assoc data-source :table-schema table-schema)]- (transforms.util/create-table-from-schema! driver db-id table-schema)-+(defn- table-schema [table-name metadata]+ {:name (if (keyword? table-name) table-name (keyword table-name))+ :columns (mapv (fn [{:keys [name base_type #_database_type]}]+ {:name name+ :type (python-runner/restricted-insert-type base_type)+ ;; :database-type database_type+ :nullable? true})+ (:fields metadata))})++(defn- insert-data!+ "Insert data from source into an existing table."+ [driver db-id table-schema data-source]+ (let [data-source (assoc data-source :table-schema table-schema)](->> #(driver/insert-from-source! driver db-id table-schema data-source)(maybe-retry-with-backoff driver))))+(defn- create-table-and-insert-data!+ "Create a table from metadata and insert data from source."+ [driver db-id table-schema data-source]+ (transforms.util/create-table-from-schema! driver db-id table-schema)+ (insert-data! driver db-id table-schema data-source))+(defn- transfer-with-rename-tables-strategy!"Transfer data using the rename-tables*! multimethod with atomicity guarantees.Creates new table, then atomically renames target->old and new->target, then drops old."@@ -168,7 +174,7 @@(log/info "Using rename-tables strategy with atomicity guarantees")(try- (create-table-and-insert-data! driver db-id source-table-name metadata data-source)+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)(transforms.util/rename-tables! driver db-id {table-name temp-table-namesource-table-name table-name})(transforms.util/drop-table! driver db-id temp-table-name)@@ -188,7 +194,7 @@(log/info "Using create-drop-rename strategy to minimize downtime")(try- (create-table-and-insert-data! driver db-id source-table-name metadata data-source)+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)(transforms.util/drop-table! driver db-id table-name)(driver/rename-table! driver db-id source-table-name table-name)@@ -207,13 +213,37 @@(try(transforms.util/drop-table! driver db-id table-name)- (create-table-and-insert-data! driver db-id table-name metadata data-source)+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)(catch Exception e(log/error e "Failed to transfer data using drop-create fallback strategy")(throw e))))-(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]+(defmulti ^:private transfer-file-to-db+ {:arglists '([driver db transform metadata temp-file])}+ (fn [_ _ transform _ _] (-> transform :target :type keyword)))++(defmethod transfer-file-to-db :table-incremental+ [driver {db-id :id}+ {:keys [target] :as transform}+ metadata temp-file]+ (let [table-name (transforms.util/qualified-table-name driver target)+ table-exists? (transforms.util/target-table-exists? transform)+ data-source {:type :jsonl-file+ :file temp-file}]++ ;; once we have more than just append, dispatch on :target-incremental-strategy++ (if (not table-exists?)+ (do+ (log/info "New table")+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))+ (insert-data! driver db-id (table-schema table-name metadata) data-source))))++(defmethod transfer-file-to-db :table+ [driver {db-id :id :as db}+ {:keys [target] :as transform}+ metadata temp-file](let [table-name (transforms.util/qualified-table-name driver target)table-exists? (transforms.util/target-table-exists? transform)data-source {:type :jsonl-file@@ -222,7 +252,7 @@(not table-exists?)(do(log/info "New table")- (create-table-and-insert-data! driver db-id table-name metadata data-source))+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))(driver.u/supports? driver :atomic-renames db)(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)@@ -253,10 +283,11 @@server-url (transforms-python.settings/python-runner-url)_ (python-runner/copy-tables-to-s3! {:run-id run-id:shared-storage @shared-storage-ref- :table-name->id (:source-tables source)- :cancel-chan cancel-chan})+ :source source+ :cancel-chan cancel-chan+ :limit (:limit source)+ :transform-id (:id transform)})_ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan-{:keys [status body] :as response}(python-runner/execute-python-code-http-call!{:server-url server-urldiff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex c12c1542741..c7c97850881 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -7,8 +7,12 @@[metabase-enterprise.transforms-python.s3 :as s3][metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase-enterprise.transforms.util :as transforms.u][metabase.analytics.prometheus :as prometheus][metabase.config.core :as config]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp][metabase.query-processor.pipeline :as qp.pipeline][metabase.util.i18n :as i18n]@@ -81,8 +85,7 @@`maybe-fixup-value` before JSON encoding."[^OutputStream os fields-meta {cols-meta :cols}](let [filtered-col-meta (m/index-by :name fields-meta)- col-names (map :name cols-meta)- none? (volatile! true)]+ col-names (map :name cols-meta)](fn([](-> os@@ -90,9 +93,6 @@(BufferedWriter.)))([^BufferedWriter writer]- ;; Workaround for LocalStack, which doesn't support zero byte files.- (when @none?- (.write writer " "))(doto writer(.flush)(.close)))@@ -105,32 +105,31 @@(map (fn [[n v]](maybe-fixup-value (filtered-col-meta n) v)))(zipmap (filter filtered-col-meta col-names)))]- (when @none? (vreset! none? false))(json/encode-to row-map writer {})(doto writer(.newLine)))))))(defn- execute-mbql-query- [db-id query rff cancel-chan]+ [query rff cancel-chan];; if we have a cancel-chan (a promise channel) for the transform, we'd like for QP to respect it;; and early exit if a value is delivered, but QP closes it when it's done. So we copy it.(with-bindings* (cond-> {}cancel-chan(assoc #'qp.pipeline/*canceled-chan* (a/go (a/<! cancel-chan))))(^:once fn* []- (qp/process-query {:type :query :database db-id :query query} rff))))+ (qp/process-query query rff))))(defn- throw-if-cancelled [cancel-chan](when (a/poll! cancel-chan)(throw (ex-info "Run cancelled" {:error-type :cancelled}))))-(defn- write-table-data-to-file! [{:keys [db-id table-id fields-meta temp-file cancel-chan limit]}]+(defn- write-query-data-to-file! [{:keys [query fields-meta temp-file cancel-chan]}](with-open [os (io/output-stream temp-file)]- (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))- rff (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))]- (execute-mbql-query db-id query rff cancel-chan)- (some-> cancel-chan throw-if-cancelled)- nil)))+ (execute-mbql-query query+ (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))+ cancel-chan)+ (some-> cancel-chan throw-if-cancelled)+ nil))(defn restricted-insert-type"Type for insertion restricted to supported"@@ -267,17 +266,31 @@:nfc_path nil{:order-by [[:database_position :asc]]}))+(defn- build-table-query+ "Build a mbql query for table, might add a proper filter for incremental transforms."+ [table-id source-incremental-strategy transform-id limit]+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)+ metadata-provider (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table metadata-provider table-id)]+ (cond-> (lib/query metadata-provider table-metadata)+ source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform-id))+ limit (lib/limit limit))))+;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.(defn copy-tables-to-s3!"Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).Blocks until all tables are fully written and committed to shared storage."[{:keys [run-idshared-storage- table-name->id+ sourcecancel-chan- limit]}]+ limit+ transform-id]}]+ (when (and (:source-incremental-strategy source)+ (> (count (:source-tables source)) 1))+ (throw (ex-info "Incremental transforms for python only supports one source table" {})));; TODO there's scope for some parallelism here, in particular across different databases- (doseq [[table-name table-id] table-name->id+ (doseq [[table-name table-id] (:source-tables source):let [{:keys [s3-client bucket-name objects]} shared-storage{data-path :path} (get objects [:table table-id :data]){manifest-path :path} (get objects [:table table-id :manifest])]]@@ -289,14 +302,12 @@fields-meta (fields-metadata driver table-id)manifest (generate-manifest table-id fields-meta)](transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]- (write-table-data-to-file!- {:db-id db-id- :driver driver- :table-id table-id- :fields-meta fields-meta- :temp-file tmp-data-file- :cancel-chan cancel-chan- :limit limit}))+ (let [query (build-table-query table-id (:source-incremental-strategy source) transform-id limit)]+ (write-query-data-to-file!+ {:query query+ :fields-meta fields-meta+ :temp-file tmp-data-file+ :cancel-chan cancel-chan})))(with-open [writer (io/writer tmp-meta-file)](json/encode-to manifest writer {}))(let [data-size (.length tmp-data-file)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.cljindex 22cc6c01523..e244fab72b1 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj@@ -179,9 +179,12 @@(defn upload-file"Upload the given file to s3"- [^S3Client s3-client ^String bucket-name ^String key ^File file]- (let [^PutObjectRequest request (put-object-request bucket-name key)]- (.putObject s3-client request (RequestBody/fromFile file))))+ [^S3Client s3 ^String bucket ^String key ^File file]+ (let [^PutObjectRequest req (put-object-request bucket key)+ body (if (zero? (.length file))+ (RequestBody/empty)+ (RequestBody/fromFile file))]+ (.putObject s3 req body)))(defn open-object"Get back the contents of the given key as a InputStream."diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex 6abfba61e9e..fb632bcdf48 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -912,3 +912,85 @@:schema (get-test-schema):name table-name}}](test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))++(deftest ^:parallel extract-columns-from-query-test+ (testing "POST /api/ee/transform/extract-columns"+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/table)+ :clickhouse :redshift :bigquery-cloud-sdk)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (letfn [(make-native-query [sql]+ {:lib/type :mbql/query+ :database (mt/id)+ :stages [{:lib/type :mbql.stage/native+ :native sql}]})]+ (testing "Successfully extracts columns from a simple SELECT query"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"+ {:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]+ (is (= ["id" "name" "category" "price"] (:columns response)))))++ (testing "Returns nil for invalid SQL"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"+ {:query (make-native-query "SELECT * FORM invalid_table")})]+ (is (nil? (:columns response)))))++ (testing "Extracts columns from query with aliases"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"+ {:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]+ (is (= ["product_id" "product_name"] (:columns response)))))++ (testing "Requires superuser permissions"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "ee/transform/extract-columns"+ {:query (make-native-query "SELECT * FROM transforms_products")}))))++ (testing "Returns 404 for non-existent database"+ (is (= "Not found."+ (mt/user-http-request :crowberto :post 404 "ee/transform/extract-columns"+ {:query {:lib/type :mbql/query+ :database 999999+ :stages [{:lib/type :mbql.stage/native+ :native "SELECT * FROM transforms_products"}]}}))))))))))++(deftest ^:parallel is-simple-query-test+ (testing "POST /api/ee/transform/is-simple-query"+ (mt/with-premium-features #{:transforms}+ (testing "Returns true for simple SELECT queries"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT id, name FROM products"})]+ (is (true? (:is_simple response)))+ (is (nil? (:reason response)))))++ (testing "Returns true for simple SELECT with WHERE clause"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT id, name FROM products WHERE category = 'Electronics'"})]+ (is (true? (:is_simple response)))))++ (testing "Returns true for simple SELECT with JOIN"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT p.id, p.name, c.name FROM products p JOIN categories c ON p.category_id = c.id"})]+ (is (true? (:is_simple response)))))++ (testing "Returns false for query with LIMIT"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT id, name FROM products LIMIT 10"})]+ (is (false? (:is_simple response)))+ (is (= "Contains a LIMIT" (:reason response)))))++ (testing "Returns false for query with OFFSET"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT id, name FROM products OFFSET 5"})]+ (is (false? (:is_simple response)))+ (is (= "Contains an OFFSET" (:reason response)))))++ (testing "Returns false for query with LIMIT and OFFSET"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "SELECT id, name FROM products LIMIT 10 OFFSET 5"})]+ (is (false? (:is_simple response)))+ (is (= "Contains a LIMIT" (:reason response)))))++ (testing "Returns false for query with CTE"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/is-simple-query"+ {:query "WITH category_counts AS (SELECT category, COUNT(*) as cnt FROM products GROUP BY category) SELECT * FROM category_counts"})]+ (is (false? (:is_simple response)))+ (is (= "Contains a CTE" (:reason response))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljnew file mode 100644index 00000000000..9cdef5a6fbc--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -0,0 +1,554 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms.incremental-test+ "Tests for incremental transforms functionality."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [honey.sql :as sql]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms.interface :as transforms.i]+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [metabase-enterprise.transforms.util :as transforms.u]+ [metabase.driver :as driver]+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sql.util :as sql.u]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor :as qp]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(def checkpoint-configs+ {:integer {:field-name "id"+ :template-tag-type :number+ :lib-column-key "column-unique-key-v1$id"+ :expected-initial-checkpoint 10+ :expected-second-checkpoint 16}+ :float {:field-name "price"+ :template-tag-type :number+ :lib-column-key "column-unique-key-v1$price"+ :expected-initial-checkpoint 49.99+ :expected-second-checkpoint 199.99}+ :temporal {:field-name "created_at"+ :template-tag-type :text+ :lib-column-key "column-unique-key-v1$created_at"+ :expected-initial-checkpoint "2024-01-10"+ :expected-second-checkpoint "2024-01-16"}})++(defn- valid-checkpoint-transform-combo?+ "Check if a checkpoint type and transform type combination is valid. "+ [checkpoint-type transform-type]+ (not (and (= checkpoint-type :temporal)+ (= transform-type :native))))++(defn- build-table-query-with-order-by+ "Wraps the original build-table-query to add ORDER BY for deterministic test results."+ [original-fn table-id source-incremental-strategy transform-id limit checkpoint-field]+ (let [query (original-fn table-id source-incremental-strategy transform-id limit)+ db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)+ metadata-provider (lib-be/application-database-metadata-provider db-id)+ ;; Find the checkpoint field in the table metadata+ checkpoint-col (->> (lib.metadata/fields metadata-provider table-id)+ (filter #(= (:name %) checkpoint-field))+ first)]+ (if checkpoint-col+ (lib/order-by query checkpoint-col)+ query)))++(defmacro with-python-order-by!+ "Execute body with python-runner/build-table-query redef'd to include ORDER BY for deterministic test results."+ [checkpoint-field & body]+ `(let [original-fn# @#'python-runner/build-table-query]+ (with-redefs [python-runner/build-table-query+ (fn [table-id# source-incremental-strategy# transform-id# limit#]+ (build-table-query-with-order-by original-fn# table-id# source-incremental-strategy# transform-id# limit# ~checkpoint-field))]+ ~@body)))++(defn- execute-transform-with-ordering!+ "Execute a transform, wrapping Python transforms with ORDER BY redefinition for deterministic results."+ [transform transform-type checkpoint-field run-opts]+ (if (= transform-type :python)+ (with-python-order-by! checkpoint-field+ (transforms.i/execute! transform run-opts))+ (transforms.i/execute! transform run-opts)))++(defn- make-incremental-source-query+ "Create a native query with optional checkpoint template tag."+ [schema checkpoint-config]+ (let [{:keys [field-name template-tag-type]} checkpoint-config+ timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))+ query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"+ timestamp-sql+ (if schema+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")+ "transforms_products")+ field-name field-name)]+ {:database (mt/id)+ :type :native+ :native {:query query+ :template-tags {"checkpoint" {:id "checkpoint"+ :name "checkpoint"+ :display-name "Checkpoint"+ :type template-tag-type+ :required false}}}}))++(defn- make-incremental-source-query-without-template-tag+ "Create a native query without template tags for testing automatic checkpoint insertion. "+ [schema]+ (let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))+ query (format "SELECT *, %s AS load_timestamp FROM %s"+ timestamp-sql+ (if schema+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")+ "transforms_products"))]+ {:database (mt/id)+ :type :native+ :native {:query query}}))++(defn- make-incremental-mbql-query+ "Create an MBQL query for incremental transforms. "+ [checkpoint-config]+ (let [{:keys [field-name]} checkpoint-config]+ (mt/mbql-query transforms_products {:expressions {"load_timestamp" [:now]}+ :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]+ :limit 10})))++(def incremental-python-body+ (str "import pandas as pd\n"+ "from datetime import datetime\n"+ "\n"+ "def transform(transforms_products):\n"+ " transforms_products['load_timestamp'] = datetime.now()\n"+ " return transforms_products"))++(defn- make-incremental-transform-payload+ "Create a transform payload for incremental transform testing.++ transform-type can be :native, :native-auto-wrap, :mbql, or :python+ checkpoint-config should be from checkpoint-configs"+ [transform-name target-table-name transform-type checkpoint-config]+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ {:keys [field-name lib-column-key]} checkpoint-config]+ {:name transform-name+ :source (case transform-type+ :native {:type "query"+ :query (make-incremental-source-query schema checkpoint-config)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter field-name}}+ :native-auto-wrap {:type "query"+ :query (make-incremental-source-query-without-template-tag schema)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter field-name}}+ :mbql {:type "query"+ :query (make-incremental-mbql-query checkpoint-config)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-unique-key lib-column-key}}+ :python {:type "python"+ :source-tables {"transforms_products" (mt/id :transforms_products)}+ :limit 10+ :body incremental-python-body+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-unique-key lib-column-key}})+ :target {:type "table-incremental"+ :schema schema+ :name target-table-name+ :database (mt/id)+ :target-incremental-strategy {:type "append"}}}))++(defn- get-table-row-count+ "Get the row count of a table by name."+ [table-name]+ (let [table (t2/select-one :model/Table :name table-name)+ mp (mt/metadata-provider)+ table-metadata (lib.metadata/table mp (:id table))+ count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))+ result (qp/process-query count-query)]+ (some-> result :data :rows first first bigint)))++(defn- get-distinct-timestamp-count+ "Get the count of distinct load_timestamp values in a table."+ [table-name]+ (let [table (t2/select-one :model/Table :name table-name)+ native-query {:database (mt/id)+ :type :native+ :native {:query (format "SELECT COUNT(DISTINCT load_timestamp) FROM %s"+ (sql.u/quote-name driver/*driver* :table+ (:schema table)+ table-name))}}+ result (qp/process-query native-query)]+ (some-> result :data :rows first first bigint)))++(defn get-checkpoint-value [transform-id]+ (#'transforms.u/next-checkpoint-value+ (transforms.u/next-checkpoint transform-id)))++(defn- compare-checkpoint-values+ "Compare two checkpoint values with type-appropriate logic. "+ [checkpoint-type expected actual]+ (case checkpoint-type+ :integer (= expected actual)+ :float (and (number? actual)+ (< (Math/abs (- expected actual)) 0.01))+ :temporal (and (string? actual)+ (str/starts-with? actual expected))))++(defn- insert-test-products!+ "Insert new products into the transforms_products table."+ [products]+ (let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ schema-prefix (if schema (str schema ".") "")+ values-list (str/join ", "+ (map (fn [{:keys [name category price created-at]}]+ (format "('%s', '%s', %s, '%s')" name category price created-at))+ products))+ insert-sql (format "INSERT INTO %s%s (name, category, price, created_at) VALUES %s"+ schema-prefix+ source-table-name+ values-list)]+ (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))++(defn- delete-test-products!+ [products]+ (let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ schema-prefix (if schema (str schema ".") "")+ delete-sql (format "DELETE FROM %s%s WHERE name IN (%s)"+ schema-prefix+ source-table-name+ (str/join ", " (map (constantly "?") products)))]+ (driver/execute-raw-queries! driver/*driver* spec [[delete-sql (mapv :name products)]])))++(defmacro with-insert-test-products! [products & body]+ `(let [products# ~products]+ (try+ (insert-test-products! products#)+ ~@body+ (finally+ (delete-test-products! products#)))))++(set! *warn-on-reflection* true)++(defn- test-drivers []+ (disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))++(deftest create-incremental-transform-test+ (testing "Creating an incremental transform with checkpoint strategy"+ (doseq [checkpoint-type [:integer :float]+ :let [transform-type :native]]+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "incremental_test"]+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)+ {:keys [field-name]} checkpoint-config+ transform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)]+ (testing "Transform is created successfully"+ (mt/with-temp [:model/Transform transform transform-payload]+ (is (some? (:id transform)))+ (is (= "Test Incremental Transform" (:name transform)))+ (is (= "table-incremental" (-> transform :target :type)))+ (is (= "checkpoint" (-> transform :source :source-incremental-strategy :type)))+ (is (= field-name (-> transform :source :source-incremental-strategy :checkpoint-filter)))++ (testing "No checkpoint exists initially"+ (is (nil? (get-checkpoint-value (:id transform)))))++ (testing "Can retrieve transform via API"+ (let [retrieved (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" (:id transform)))]+ (is (= (:id transform) (:id retrieved)))+ (is (= "Test Incremental Transform" (:name retrieved)))))++ (testing "Transform appears in list endpoint"+ (let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")+ our-transform (first (filter #(= (:id transform) (:id %)) transforms))]+ (is (some? our-transform))+ (is (= "Test Incremental Transform" (:name our-transform))))))))))))))))++(deftest run-incremental-transform-twice-test+ (testing "Running an incremental transform twice processes only new data on second run"+ (doseq [checkpoint-type [:integer :float :temporal]+ transform-type [:native :mbql :python]+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "incremental_twice"]+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)+ {:keys [expected-initial-checkpoint expected-second-checkpoint]} checkpoint-config+ transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]+ (mt/with-temp [:model/Transform transform transform-payload]+ (testing "First run processes all data"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (transforms.tu/wait-for-table target-table 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 10 row-count) "First run should process the first 10 products")+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")++ (testing "Checkpoint is created after first run"+ (let [checkpoint (get-checkpoint-value (:id transform))]+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from first 10 rows" (:field-name checkpoint-config)))))))++ (testing "Second run should process the remaining rows"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 16 row-count) "Second run should add remaining 6 rows")+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")+ (let [checkpoint (get-checkpoint-value (:id transform))]+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config))))))))))))))))++(deftest switch-incremental-to-non-incremental-test+ (testing "Switching an incremental transform to non-incremental overwrites data"+ (doseq [checkpoint-type [:integer :float :temporal]+ transform-type [:native :mbql :python]+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "switch_incr_to_non_incr"]+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)+ {:keys [expected-initial-checkpoint]} checkpoint-config+ initial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)]+ (mt/with-temp [:model/Transform transform initial-payload]+ (testing "Initial incremental run processes first batch"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (transforms.tu/wait-for-table target-table 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 10 row-count) "Initial run should process first 10 products")+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))++ (testing "Second incremental run processes remaining data"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 16 row-count) "Should have 16 rows after second incremental run")+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")))++ (testing "Switch to non-incremental via PUT API"+ (let [non-incremental-payload (-> initial-payload+ (update :source dissoc :source-incremental-strategy)+ (update :target dissoc :source-incremental-strategy)+ (update :target assoc :type "table"))+ updated (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" (:id transform))+ non-incremental-payload)]+ (is (= "table" (-> updated :target :type)))+ (is (nil? (-> updated :source :source-incremental-strategy)))))++ (testing "Non-incremental run overwrites all data"+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 10 row-count) "Should overwrite to 10 rows")+ (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))++ (testing "Running again still overwrites"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 10 row-count) "Should still have 10 rows after another run")+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))))))))))))))++(deftest switch-non-incremental-to-incremental-test+ (testing "Switching a non-incremental transform to incremental computes checkpoint from existing data"+ (doseq [checkpoint-type [:integer :float :temporal]+ transform-type [:native :mbql :python]+ :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]+ (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "switch_non_incr_to_incr"]+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)+ {:keys [expected-second-checkpoint]} checkpoint-config+ incremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)+ initial-payload (-> incremental-payload+ (update :source dissoc :source-incremental-strategy)+ (update :target dissoc :source-incremental-strategy)+ (update :target assoc :type "table"))]+ (mt/with-temp [:model/Transform transform initial-payload]+ (testing "Initial non-incremental run creates table"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (transforms.tu/wait-for-table target-table 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 10 row-count) "Initial run should process 10 products")+ (is (= 1 distinct-timestamps) "All rows should have same timestamp from non-incremental run")++ (testing "No checkpoint exists"+ (let [checkpoint (get-checkpoint-value (:id transform))]+ (is (nil? checkpoint) "No checkpoint for non-incremental transform")))))++ (testing "Switch to incremental via PUT API"+ (let [updated (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" (:id transform))+ incremental-payload)]+ (is (= "table-incremental" (-> updated :target :type)))+ (is (= "checkpoint" (-> updated :source :source-incremental-strategy :type)))))++ (testing "First incremental run after switch processes no new data"+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "Should process remaining 6 entries")+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))++ (when-not (= driver/*driver* :clickhouse)+ (testing "Add new data and run incrementally"+ (with-insert-test-products!+ [{:name "After Switch Product"+ :category "Gadget"+ :price 379.99+ :created-at "2024-01-20T10:00:00"}]++ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")+ ;; For integer checkpoints, we can verify exact value >= 17+ ;; For float/temporal, just verify checkpoint exists+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))++(deftest native-query-without-template-tag-test+ (testing "Native query without template tags uses automatic checkpoint wrapping"+ (doseq [checkpoint-type [:integer :float :temporal]]+ (testing (format "with %s checkpoint" (name checkpoint-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "native_no_template"]+ (let [checkpoint-config (get checkpoint-configs checkpoint-type)+ {:keys [expected-second-checkpoint field-name]} checkpoint-config+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ transform-payload {:name "Native Without Template Tag"+ :source {:type "query"+ :query (make-incremental-source-query-without-template-tag schema)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter field-name}}+ :target {:type "table-incremental"+ :schema schema+ :name target-table+ :database (mt/id)+ :target-incremental-strategy {:type "append"}}}]+ (mt/with-temp [:model/Transform transform transform-payload]+ (testing "First run processes all existing data"+ (transforms.i/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-table 10000)+ (let [row-count (get-table-row-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "First run should process all 16 products")+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))++ (testing "Second run without new data adds nothing"+ (transforms.i/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows, no new data")))++ (when-not (= driver/*driver* :clickhouse)+ (testing "After inserting new data, incremental run appends only new rows"+ (with-insert-test-products!+ [{:name "New Product 1"+ :category "Electronics"+ :price 299.99+ :created-at "2024-01-21T10:00:00"}+ {:name "New Product 2"+ :category "Books"+ :price 319.99+ :created-at "2024-01-21T11:00:00"}]++ (transforms.i/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")+ (is (some? checkpoint) "Checkpoint should be updated")))))))))))))))++(deftest ^:postgres-only native-query-with-temporal-checkpoint-test+ (testing "Native query with temporal checkpoint"+ ;; we test only in postgres because it's easy to cast to ::timestamp+ (mt/test-drivers [:postgres]+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "native_temporal_cast"]+ (let [checkpoint-config (get checkpoint-configs :temporal)+ {:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-config+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))+ query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}::timestamp]] ORDER BY %s LIMIT 10"+ timestamp-sql+ (if schema+ (sql.u/quote-name driver/*driver* :table schema "transforms_products")+ "transforms_products")+ field-name+ field-name)+ transform-payload {:name "Native With Temporal Cast"+ :source {:type "query"+ :query {:database (mt/id)+ :type :native+ :native {:query query+ :template-tags {"checkpoint" {:id "checkpoint"+ :name "checkpoint"+ :display-name "Checkpoint"+ :type :text+ :required false}}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter field-name}}+ :target {:type "table-incremental"+ :schema schema+ :name target-table+ :database (mt/id)+ :target-incremental-strategy {:type "append"}}}]+ (mt/with-temp [:model/Transform transform transform-payload]+ (testing "First run processes first batch"+ (transforms.i/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table target-table 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 10 row-count) "First run should process the first 10 products")+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")+ (is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))++ (testing "Second run processes remaining data"+ (transforms.i/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "Second run should add remaining 6 rows")+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")+ (is (compare-checkpoint-values :temporal expected-second-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))++ (testing "Third run without new data adds nothing"+ (transforms.i/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows, no new data")))++ (testing "After inserting new data, incremental run appends only new rows"+ (with-insert-test-products!+ [{:name "New Temporal Product"+ :category "Electronics"+ :price 299.99+ :created-at "2024-01-21T10:00:00"}]++ (transforms.i/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljindex b36ea5efb50..1761bacf0ed 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -1,13 +1,11 @@(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.drivers-test"Comprehensive tests for Python transforms across all supported drivers with all base and exotic types."(:require- [clojure.core.async :as a][clojure.string :as str][clojure.test :refer :all][metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms-python.python-runner :as python-runner]- [metabase-enterprise.transforms-python.s3 :as s3]- [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms-python.python-runner-test :as python-runner-test][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver]@@ -23,32 +21,6 @@(set! *warn-on-reflection* true)-(def test-id 42)--(defn- execute!- "Execute a Python transform with the given code and tables"- [{:keys [code tables]}]- (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]- (let [server-url (transforms-python.settings/python-runner-url)- cancel-chan (a/promise-chan)- table-name->id (or tables {})- _ (python-runner/copy-tables-to-s3! {:run-id test-id- :shared-storage @shared-storage-ref- :table-name->id table-name->id- :cancel-chan cancel-chan})- response (python-runner/execute-python-code-http-call! {:server-url server-url- :code code- :run-id test-id- :table-name->id table-name->id- :shared-storage @shared-storage-ref})- events (python-runner/read-events @shared-storage-ref)- output-manifest (python-runner/read-output-manifest @shared-storage-ref)]- (merge (:body response)- {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))- :output-manifest output-manifest- :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))- :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))-(defn- execute-e2e-transform!"Execute an e2e Python transform test using execute-python-transform!"[table-name transform-code source-tables]@@ -245,8 +217,8 @@(defn- execute-and-validate-transform!"Execute a Python transform and validate the output, returning validation results."[transform-code table-name table-id expected-columns expected-row-count]- (let [result (execute! {:code transform-code- :tables {table-name table-id}})]+ (let [result (python-runner-test/execute! {:code transform-code+ :tables {table-name table-id}})](validate-transform-output result expected-columns expected-row-count)))(defn- test-exotic-types-for-driver!@@ -409,11 +381,11 @@" df['name_upper'] = df['name'].str.upper()\n"" return df");; Run the transform twice- result1 (execute!+ result1 (python-runner-test/execute!{:code transform-code:tables {table-name table-id}})- result2 (execute!+ result2 (python-runner-test/execute!{:code transform-code:tables {table-name table-id}})](testing "Both transforms succeeded"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljindex c542983351c..dbc045a95d9 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -55,7 +55,7 @@(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))-(defn- execute! [{:keys [code tables]}]+(defn execute! [{:keys [code tables]}](with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))](let [server-url (transforms-python.settings/python-runner-url)cancel-chan (a/promise-chan)@@ -63,7 +63,7 @@test-id (next-job-run-id)_ (python-runner/copy-tables-to-s3! {:run-id test-id:shared-storage @shared-storage-ref- :table-name->id table-name->id+ :source {:source-tables table-name->id}:cancel-chan cancel-chan})response (python-runner/execute-python-code-http-call! {:server-url server-url:code codediff --git a/enterprise/frontend/src/metabase-enterprise/api/transform.ts b/enterprise/frontend/src/metabase-enterprise/api/transform.tsindex 45345abca88..92e29222e4a 100644--- a/enterprise/frontend/src/metabase-enterprise/api/transform.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/transform.ts@@ -1,6 +1,10 @@import { isResourceNotFoundError } from "metabase/lib/errors";import type {+ CheckQueryComplexityRequest,+ CheckQueryComplexityResponse,CreateTransformRequest,+ ExtractColumnsFromQueryRequest,+ ExtractColumnsFromQueryResponse,ListTransformRunsRequest,ListTransformRunsResponse,ListTransformsRequest,@@ -189,6 +193,26 @@ export const transformApi = EnterpriseApi.injectEndpoints({invalidatesTags: (_, error) =>invalidateTags(error, [listTag("transform"), listTag("table")]),}),+ extractColumnsFromQuery: builder.mutation<+ ExtractColumnsFromQueryResponse,+ ExtractColumnsFromQueryRequest+ >({+ query: (body) => ({+ method: "POST",+ url: "/api/ee/transform/extract-columns",+ body,+ }),+ }),+ checkQueryComplexity: builder.query<+ CheckQueryComplexityResponse,+ CheckQueryComplexityRequest+ >({+ query: (queryString) => ({+ method: "POST",+ url: "/api/ee/transform/is-simple-query",+ body: { query: queryString },+ }),+ }),}),});@@ -204,4 +228,6 @@ export const {useUpdateTransformMutation,useDeleteTransformMutation,useDeleteTransformTargetMutation,+ useExtractColumnsFromQueryMutation,+ useLazyCheckQueryComplexityQuery,} = transformApi;diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsxnew file mode 100644index 00000000000..21ce0e2bb48--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx@@ -0,0 +1,256 @@+import { useFormikContext } from "formik";+import { useEffect, useMemo, useState } from "react";+import { t } from "ttag";++import { FormSelect, FormSwitch } from "metabase/forms";+import { useSelector } from "metabase/lib/redux";+import { getMetadata } from "metabase/selectors/metadata";+import { Alert, Stack } from "metabase/ui";+import { useLazyCheckQueryComplexityQuery } from "metabase-enterprise/api";+import {+ KeysetColumnSelect,+ PythonKeysetColumnSelect,+} from "metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect";+import { NativeQueryColumnSelect } from "metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect";+import {+ CHECKPOINT_TEMPLATE_TAG,+ SOURCE_STRATEGY_OPTIONS,+ TARGET_STRATEGY_OPTIONS,+} from "metabase-enterprise/transforms/constants";+import type { NewTransformValues } from "metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal";+import * as Lib from "metabase-lib";+import Question from "metabase-lib/v1/Question";+import type { TransformSource } from "metabase-types/api";++export const IncrementalTransformSettings = ({+ source,+ checkOnMount,+}: {+ source: TransformSource;+ checkOnMount?: boolean;+}) => {+ const metadata = useSelector(getMetadata);+ const { values } = useFormikContext<NewTransformValues>();+ // Convert DatasetQuery to Lib.Query via Question+ const libQuery = useMemo(() => {+ if (source.type !== "query") {+ return null;+ }++ try {+ const question = Question.create({+ dataset_query: source.query,+ metadata,+ });+ return question.query();+ } catch (error) {+ console.error("SourceStrategyFields: Error creating question", error);+ return null;+ }+ }, [source, metadata]);++ // Check if this is an MBQL query (not native SQL or Python)+ const isMbqlQuery = useMemo(() => {+ if (!libQuery) {+ return false;+ }++ try {+ const queryDisplayInfo = Lib.queryDisplayInfo(libQuery);+ return !queryDisplayInfo.isNative;+ } catch (error) {+ console.error("Error checking query type", error);+ return false;+ }+ }, [libQuery]);++ // Check if this is a Python transform with exactly one source table+ // Incremental transforms are only supported for single-table Python transforms+ const isPythonTransform = source.type === "python";+ const isMultiTablePythonTransform =+ isPythonTransform && Object.keys(source["source-tables"]).length > 1;++ const [checkQueryComplexity, { data: complexity }] =+ useLazyCheckQueryComplexityQuery();+ const [showComplexityWarning, setShowComplexityWarning] = useState(false);++ const { transformType, query } = useMemo(() => {+ if (isMbqlQuery) {+ return { transformType: "query" as const, query: libQuery };+ }+ if (isPythonTransform) {+ return { transformType: "python" as const, query: null };+ }+ return { transformType: "native" as const, query: libQuery };+ }, [isMbqlQuery, isPythonTransform, libQuery]);++ const hasCheckpointTag = useMemo(() => {+ if (!libQuery || transformType !== "native") {+ return false;+ }+ const tags = Lib.templateTags(libQuery);+ return CHECKPOINT_TEMPLATE_TAG in tags;+ }, [libQuery, transformType]);++ useEffect(() => {+ async function checkExistingQueryComplexity() {+ if (+ checkOnMount &&+ transformType === "native" &&+ libQuery &&+ !hasCheckpointTag &&+ "source-incremental-strategy" in source+ ) {+ const { is_simple } = await checkQueryComplexity(+ Lib.rawNativeQuery(libQuery),+ true,+ ).unwrap();+ setShowComplexityWarning(!is_simple);+ }+ }+ checkExistingQueryComplexity();+ }, [+ checkOnMount,+ checkQueryComplexity,+ libQuery,+ transformType,+ source,+ hasCheckpointTag,+ ]);++ return (+ <>+ <FormSwitch+ disabled={isMultiTablePythonTransform}+ name="incremental"+ label={+ isMultiTablePythonTransform+ ? t`Incremental transforms are only supported for single data source transforms.`+ : t`Incremental?`+ }+ onChange={async (e) => {+ if (+ e.target.checked &&+ transformType === "native" &&+ !hasCheckpointTag &&+ query+ ) {+ const complexity = await checkQueryComplexity(+ Lib.rawNativeQuery(query),+ true,+ ).unwrap();+ setShowComplexityWarning(complexity?.is_simple === false);+ }+ }}+ />+ {values?.incremental && (+ <>+ {showComplexityWarning && (+ <Alert variant="info" icon="info">+ <Stack gap="xs">+ <span>+ {t`This query is too complex to allow automatic checkpoint column selection. You may need to explicitely add a conditional filter in your query, for example:`}+ </span>+ <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>+ <span>+ {t`Reason: `}+ <strong>{complexity?.reason}</strong>+ </span>+ </Stack>+ </Alert>+ )}+ <SourceStrategyFields+ source={source}+ query={query}+ type={transformType}+ />+ <TargetStrategyFields />+ </>+ )}+ </>+ );+};++function TargetStrategyFields() {+ const { values } = useFormikContext<NewTransformValues>();++ if (!values.incremental) {+ return null;+ }++ return (+ <>+ {TARGET_STRATEGY_OPTIONS.length > 1 && (+ <FormSelect+ name="targetStrategy"+ label={t`Target Strategy`}+ description={t`How to update the target table`}+ data={TARGET_STRATEGY_OPTIONS}+ />+ )}+ {/* Append strategy has no additional fields */}+ {/* Future strategies like "merge" could add fields here */}+ </>+ );+}++type SourceStrategyFieldsProps = {+ source: TransformSource;+ query: Lib.Query | null;+ type: "query" | "native" | "python";+};++function SourceStrategyFields({+ source,+ query,+ type,+}: SourceStrategyFieldsProps) {+ const { values } = useFormikContext<NewTransformValues>();+ if (!values.incremental) {+ return null;+ }++ return (+ <>+ {SOURCE_STRATEGY_OPTIONS.length > 1 && (+ <FormSelect+ name="sourceStrategy"+ label={t`Source Strategy`}+ description={t`How to track which rows to process`}+ data={SOURCE_STRATEGY_OPTIONS}+ />+ )}+ {values.sourceStrategy === "checkpoint" && (+ <>+ {type === "query" && query && (+ <KeysetColumnSelect+ name="checkpointFilterUniqueKey"+ label={t`Source Filter Field`}+ placeholder={t`Select a field to filter on`}+ description={t`Which field from the source to use in the incremental filter`}+ query={query}+ />+ )}+ {type === "native" && query && (+ <NativeQueryColumnSelect+ name="checkpointFilter"+ label={t`Source Filter Field`}+ placeholder={t`e.g. id, created_at`}+ description={t`Column to use in the incremental filter`}+ query={query}+ />+ )}+ {type === "python" && "source-tables" in source && (+ <PythonKeysetColumnSelect+ name="checkpointFilterUniqueKey"+ label={t`Source Filter Field`}+ placeholder={t`Select a field to filter on`}+ description={t`Which field from the source to use in the incremental filter`}+ sourceTables={source["source-tables"]}+ />+ )}+ </>+ )}+ </>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsxnew file mode 100644index 00000000000..cccff014020--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx@@ -0,0 +1,88 @@+import { useMemo } from "react";++import { FormSelect } from "metabase/forms";+import type { SelectOption } from "metabase/ui";+import * as Lib from "metabase-lib";++type KeysetColumnSelectProps = {+ name: string;+ label: string;+ placeholder: string;+ description: string;+ query: Lib.Query | null;+ disabled?: boolean;+};++export function KeysetColumnSelect({+ name,+ label,+ placeholder,+ description,+ query,+ disabled,+}: KeysetColumnSelectProps) {+ const columnOptions = useMemo((): Array<SelectOption> => {+ if (!query) {+ return [];+ }++ try {+ // Use -1 to get the last stage+ const stageIndex = -1;++ const returnedColumns = Lib.returnedColumns(query, stageIndex);+ const filterableColumns = Lib.filterableColumns(query, stageIndex);++ const filterableIdentifiers = new Set(+ filterableColumns.map((col) => {+ return Lib.columnKey(col);+ }),+ );++ const numericFilterableColumns = returnedColumns.filter((column) => {+ return (+ filterableIdentifiers.has(Lib.columnKey(column)) &&+ (Lib.isNumeric(column) || Lib.isTemporal(column))+ );+ });++ // Convert to select options with unique keys+ const seenKeys = new Set<string>();+ const uniqueColumns: Array<SelectOption> = [];++ numericFilterableColumns.forEach((column) => {+ const columnInfo = Lib.displayInfo(query, stageIndex, column);+ const uniqueKey = Lib.columnUniqueKey(column);++ if (!seenKeys.has(uniqueKey)) {+ seenKeys.add(uniqueKey);++ const label = columnInfo.longDisplayName;++ uniqueColumns.push({ value: uniqueKey, label });+ }+ });++ return uniqueColumns;+ } catch (error) {+ // If we can't extract columns (e.g., invalid query), return empty array+ console.error(+ "KeysetColumnSelect: Error extracting columns from query:",+ error,+ );+ return [];+ }+ }, [query]);++ return (+ <FormSelect+ name={name}+ label={label}+ placeholder={placeholder}+ description={description}+ data={columnOptions}+ searchable+ disabled={disabled || columnOptions.length === 0}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsxnew file mode 100644index 00000000000..db49a96cf31--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx@@ -0,0 +1,71 @@+import { skipToken } from "@reduxjs/toolkit/query";+import { useMemo } from "react";++import { useGetTableQueryMetadataQuery } from "metabase/api";+import { useSelector } from "metabase/lib/redux";+import { getMetadata } from "metabase/selectors/metadata";+import * as Lib from "metabase-lib";+import type { PythonTransformTableAliases } from "metabase-types/api";++import { KeysetColumnSelect } from "./KeysetColumnSelect";++type PythonKeysetColumnSelectProps = {+ name: string;+ label: string;+ placeholder: string;+ description: string;+ sourceTables: PythonTransformTableAliases;+};++export function PythonKeysetColumnSelect({+ name,+ label,+ placeholder,+ description,+ sourceTables,+}: PythonKeysetColumnSelectProps) {+ const metadata = useSelector(getMetadata);++ // Get the first (and should be only) table ID+ // Incremental transforms are only supported for single-table Python transforms+ const tableId = useMemo(() => {+ const tableIds = Object.values(sourceTables);+ return tableIds.length === 1 ? tableIds[0] : null;+ }, [sourceTables]);++ // Fetch metadata for the table+ const {+ data: table,+ isLoading,+ error,+ } = useGetTableQueryMetadataQuery(tableId ? { id: tableId } : skipToken);++ // Create a query from the table to get column metadata+ const query = useMemo(() => {+ if (!table || !table.db_id) {+ return null;+ }++ try {+ const metadataProvider = Lib.metadataProvider(table.db_id, metadata);+ const tableMetadata = Lib.tableOrCardMetadata(metadataProvider, table.id);+ if (!tableMetadata) {+ return null;+ }+ return Lib.queryFromTableOrCardMetadata(metadataProvider, tableMetadata);+ } catch {+ return null;+ }+ }, [table, metadata]);++ return (+ <KeysetColumnSelect+ name={name}+ label={label}+ placeholder={placeholder}+ description={description}+ query={query}+ disabled={isLoading || !!error}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.tsnew file mode 100644index 00000000000..fda664b4009--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts@@ -0,0 +1,2 @@+export { KeysetColumnSelect } from "./KeysetColumnSelect";+export { PythonKeysetColumnSelect } from "./PythonKeysetColumnSelect";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsxnew file mode 100644index 00000000000..6b63e154dd8--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx@@ -0,0 +1,90 @@+import { useEffect, useState } from "react";+import { useLatest } from "react-use";++import { FormSelect, FormTextInput } from "metabase/forms";+import { Loader } from "metabase/ui";+import { useExtractColumnsFromQueryMutation } from "metabase-enterprise/api";+import * as Lib from "metabase-lib";++type NativeQueryColumnSelectProps = {+ name: string;+ label: string;+ description: string;+ placeholder: string;+ query: Lib.Query;+};++export function NativeQueryColumnSelect({+ name,+ label,+ description,+ placeholder,+ query,+}: NativeQueryColumnSelectProps) {+ const [columns, setColumns] = useState<string[] | null>(null);+ const [extractColumns, { isLoading }] = useExtractColumnsFromQueryMutation();++ const queryRef = useLatest(query);++ useEffect(() => {+ let isCancelled = false;+ if (+ columns &&+ Lib.areLegacyQueriesEqual(+ Lib.toLegacyQuery(queryRef.current),+ Lib.toLegacyQuery(query),+ )+ ) {+ return;+ }++ const extract = async () => {+ try {+ const result = await extractColumns({+ query: Lib.toJsQuery(query),+ }).unwrap();++ if (!isCancelled) {+ // Only use columns if we got at least one+ setColumns(result.columns.length > 0 ? result.columns : null);+ }+ } catch (error) {+ // On error, fall back to text input+ if (!isCancelled) {+ setColumns(null);+ }+ }+ };++ extract();++ return () => {+ isCancelled = true;+ };+ }, [extractColumns, query, queryRef, columns]);++ // If we successfully extracted columns, show a selector+ if (columns && columns.length > 0) {+ return (+ <FormSelect+ name={name}+ rightSection={isLoading ? <Loader size="sm" /> : null}+ disabled={isLoading}+ label={label}+ description={description}+ placeholder={placeholder}+ data={columns.map((col) => ({ value: col, label: col }))}+ />+ );+ }++ // Otherwise, fall back to text input+ return (+ <FormTextInput+ name={name}+ label={label}+ placeholder={placeholder}+ description={description}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.tsnew file mode 100644index 00000000000..b95d8e9d607--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts@@ -0,0 +1 @@+export { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsxindex 809e313f487..94ed1f7f280 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx@@ -47,7 +47,13 @@ export function TransformEditor({const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);const handleQueryChange = (query: Lib.Query) => {- onChangeSource({ type: "query", query: Lib.toJsQuery(query) });+ const newSource: QueryTransformSource = {+ ...source,+ type: "query",+ query: Lib.toJsQuery(query),+ };++ onChangeSource(newSource);};return (diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms/constants.tsindex bbe97338858..18107133265 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/constants.ts@@ -1,3 +1,5 @@+import { t } from "ttag";+import { isCypressActive } from "metabase/env";export const NAME_MAX_LENGTH = 254;@@ -6,3 +8,22 @@ export const POLLING_INTERVAL = isCypressActive ? 200 : 3000;export const FILTER_WIDGET_MIN_WIDTH = 300;export const FILTER_WIDGET_MAX_HEIGHT = 400;++export const CHECKPOINT_TEMPLATE_TAG = "checkpoint";+export const SOURCE_STRATEGY_OPTIONS = [+ {+ value: CHECKPOINT_TEMPLATE_TAG,+ get label() {+ return t`Checkpoint`;+ },+ },+];++export const TARGET_STRATEGY_OPTIONS = [+ {+ value: "append" as const,+ get label() {+ return t`Append`;+ },+ },+];diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsxindex 4a18760e67d..68cee1087b4 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx@@ -8,7 +8,9 @@ import {useGetDatabaseQuery,useListDatabaseSchemasQuery,} from "metabase/api";+import { getErrorMessage } from "metabase/api/utils";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { useToast } from "metabase/common/hooks";import {Form,FormErrorMessage,@@ -19,6 +21,7 @@ import {import * as Errors from "metabase/lib/errors";import { Box, Button, Group, Modal, Stack } from "metabase/ui";import { useCreateTransformMutation } from "metabase-enterprise/api";+import { IncrementalTransformSettings } from "metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings";import type {CreateTransformRequest,Transform,@@ -26,15 +29,27 @@ import type {} from "metabase-types/api";import { trackTransformCreated } from "../../../analytics";-import type { NewTransformValues } from "../types";import { SchemaFormSelect } from "./../../../components/SchemaFormSelect";-const NEW_TRANSFORM_SCHEMA = Yup.object({- name: Yup.string().required(Errors.required),- targetName: Yup.string().required(Errors.required),- targetSchema: Yup.string().nullable(),-});+function getValidationSchema() {+ return Yup.object({+ name: Yup.string().required(Errors.required),+ targetName: Yup.string().required(Errors.required),+ targetSchema: Yup.string().nullable().defined(),+ incremental: Yup.boolean().required(),+ // For native queries, use checkpointFilter (plain string)+ checkpointFilter: Yup.string().nullable(),+ // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)+ checkpointFilterUniqueKey: Yup.string().nullable(),+ sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),+ targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),+ });+}++export type NewTransformValues = Yup.InferType<+ ReturnType<typeof getValidationSchema>+>;type CreateTransformModalProps = {source: TransformSource;@@ -74,6 +89,7 @@ function CreateTransformForm({onCreate,onClose,}: CreateTransformFormProps) {+ const [sendToast] = useToast();const databaseId =source.type === "query" ? source.query.database : source["source-database"];@@ -102,6 +118,8 @@ function CreateTransformForm({[schemas, defaultValues],);+ const validationSchema = useMemo(() => getValidationSchema(), []);+if (isLoading || error != null) {return <LoadingAndErrorWrapper loading={isLoading} error={error} />;}@@ -111,21 +129,26 @@ function CreateTransformForm({throw new Error("Database ID is required");}const request = getCreateRequest(source, values, databaseId);- const transform = await createTransform(request).unwrap();-- trackTransformCreated({ transformId: transform.id });-- onCreate(transform);+ try {+ const transform = await createTransform(request).unwrap();+ trackTransformCreated({ transformId: transform.id });+ onCreate(transform);+ } catch (error) {+ sendToast({+ message: getErrorMessage(error, t`Failed to create transform`),+ icon: "warning",+ });+ }};return (<FormProviderinitialValues={initialValues}- validationSchema={NEW_TRANSFORM_SCHEMA}+ validationSchema={validationSchema}onSubmit={handleSubmit}><Form>- <Stack gap="lg">+ <Stack gap="lg" mt="sm"><FormTextInputname="name"label={t`Name`}@@ -144,6 +167,7 @@ function CreateTransformForm({label={t`Table name`}placeholder={t`descriptive_name`}/>+ <IncrementalTransformSettings source={source} /><Group><Box flex={1}><FormErrorMessage />@@ -166,22 +190,71 @@ function getInitialValues(targetName: "",targetSchema: schemas?.[0] || null,...defaultValues,+ checkpointFilter: null,+ checkpointFilterUniqueKey: null,+ incremental: false,+ sourceStrategy: "checkpoint",+ targetStrategy: "append",};}function getCreateRequest(source: TransformSource,- { name, targetName, targetSchema }: NewTransformValues,+ {+ name,+ targetName,+ targetSchema,+ incremental,+ checkpointFilter,+ checkpointFilterUniqueKey,+ sourceStrategy,+ targetStrategy,+ }: NewTransformValues,databaseId: number,): CreateTransformRequest {+ // Build the source with incremental strategy if enabled+ let transformSource: TransformSource;+ if (incremental) {+ // For native queries, use checkpoint-filter (plain string)+ // For MBQL/Python queries, use checkpoint-filter-unique-key (prefixed format)+ const strategyFields = checkpointFilter+ ? { "checkpoint-filter": checkpointFilter }+ : checkpointFilterUniqueKey+ ? { "checkpoint-filter-unique-key": checkpointFilterUniqueKey }+ : {};++ transformSource = {+ ...source,+ "source-incremental-strategy": {+ type: sourceStrategy,+ ...strategyFields,+ },+ };+ } else {+ transformSource = source;+ }++ // Build the target with incremental strategy if enabled+ const transformTarget: CreateTransformRequest["target"] = incremental+ ? {+ type: "table-incremental",+ name: targetName,+ schema: targetSchema,+ database: databaseId,+ "target-incremental-strategy": {+ type: targetStrategy,+ },+ }+ : {+ type: "table",+ name: targetName,+ schema: targetSchema ?? null,+ database: databaseId,+ };+return {- name: name,- source,- target: {- type: "table",- name: targetName,- schema: targetSchema ?? null,- database: databaseId,- },+ name,+ source: transformSource,+ target: transformTarget,};}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.tsindex b8a1ea8b56b..e69de29bb2d 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/types.ts@@ -1,5 +0,0 @@-export type NewTransformValues = {- name: string;- targetName: string;- targetSchema: string | null;-};diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.tsindex c537e3464b0..3343ca5e8eb 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts@@ -6,7 +6,7 @@ import type {SuggestedTransform,} from "metabase-types/api";-import type { NewTransformValues } from "./types";+import type { NewTransformValues } from "./CreateTransformModal/CreateTransformModal";export function getInitialQuerySource(): QueryTransformSource {const question = Question.create({ DEPRECATED_RAW_MBQL_type: "query" });diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsxindex b6f82bd14bb..1469687aa4e 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/TargetSection.tsx@@ -24,6 +24,7 @@ import type { Transform } from "metabase-types/api";import { TitleSection } from "../../../components/TitleSection";import { isTransformRunning, sourceDatabaseId } from "../../../utils";+import { UpdateIncrementalModal } from "./UpdateIncrementalModal";import { UpdateTargetModal } from "./UpdateTargetModal";type TargetSectionProps = {@@ -42,6 +43,7 @@ export function TargetSection({ transform }: TargetSectionProps) {<Divider /><Group p="lg"><EditTargetButton transform={transform} />+ <EditIncrementalButton transform={transform} /><EditMetadataButton transform={transform} /></Group></TitleSection>@@ -196,6 +198,44 @@ function EditTargetButton({ transform }: EditTargetButtonProps) {);}+type EditIncrementalButtonProps = {+ transform: Transform;+};++function EditIncrementalButton({ transform }: EditIncrementalButtonProps) {+ const [isModalOpened, { open: openModal, close: closeModal }] =+ useDisclosure();+ const { sendSuccessToast } = useMetadataToasts();++ const isIncremental = transform.target.type === "table-incremental";++ const handleUpdate = () => {+ closeModal();+ sendSuccessToast(t`Incremental settings updated`);+ };++ return (+ <>+ <Button+ leftSection={<Icon name="refresh" aria-hidden />}+ disabled={isTransformRunning(transform)}+ onClick={openModal}+ >+ {isIncremental+ ? t`Edit incremental settings`+ : t`Make transform incremental`}+ </Button>+ {isModalOpened && (+ <UpdateIncrementalModal+ transform={transform}+ onUpdate={handleUpdate}+ onClose={closeModal}+ />+ )}+ </>+ );+}+type EditMetadataButtonProps = {transform: Transform;};@@ -213,7 +253,7 @@ function EditMetadataButton({ transform }: EditMetadataButtonProps) {leftSection={<Icon name="label" aria-hidden />}data-testid="table-metadata-link">- {t`Edit this table’s metadata`}+ {t`Edit this table's metadata`}</Button>);}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsxnew file mode 100644index 00000000000..92205c5f069--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateIncrementalModal.tsx@@ -0,0 +1,189 @@+import { useMemo } from "react";+import { t } from "ttag";+import * as Yup from "yup";++import { getErrorMessage } from "metabase/api/utils";+import { useToast } from "metabase/common/hooks";+import {+ Form,+ FormErrorMessage,+ FormProvider,+ FormSubmitButton,+} from "metabase/forms";+import { Box, Button, FocusTrap, Group, Modal, Stack } from "metabase/ui";+import { useUpdateTransformMutation } from "metabase-enterprise/api";+import { IncrementalTransformSettings } from "metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings";+import type { Transform } from "metabase-types/api";++type UpdateIncrementalModalProps = {+ transform: Transform;+ onUpdate: () => void;+ onClose: () => void;+};++export function UpdateIncrementalModal({+ transform,+ onUpdate,+ onClose,+}: UpdateIncrementalModalProps) {+ return (+ <Modal+ title={t`Configure incremental settings`}+ opened+ padding="xl"+ onClose={onClose}+ >+ <FocusTrap.InitialFocus />+ <UpdateIncrementalForm+ transform={transform}+ onUpdate={onUpdate}+ onClose={onClose}+ />+ </Modal>+ );+}++type IncrementalValues = {+ incremental: boolean;+ sourceStrategy: "checkpoint";+ checkpointFilter: string | null;+ checkpointFilterUniqueKey: string | null;+ targetStrategy: "append";+};++function getValidationSchema() {+ return Yup.object({+ incremental: Yup.boolean().required(),+ sourceStrategy: Yup.string().oneOf(["checkpoint"]).required(),+ // For native queries, use checkpointFilter (plain string)+ checkpointFilter: Yup.string().nullable(),+ // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)+ checkpointFilterUniqueKey: Yup.string().nullable(),+ targetStrategy: Yup.string().oneOf(["append"]).required(),+ });+}++type UpdateIncrementalFormProps = {+ transform: Transform;+ onUpdate: () => void;+ onClose: () => void;+};++function UpdateIncrementalForm({+ transform,+ onUpdate,+ onClose,+}: UpdateIncrementalFormProps) {+ const [sendToast] = useToast();+ const [updateTransform] = useUpdateTransformMutation();+ const initialValues = useMemo(() => getInitialValues(transform), [transform]);+ const validationSchema = useMemo(() => getValidationSchema(), []);++ const handleSubmit = async (values: IncrementalValues) => {+ // Build the source with incremental strategy if enabled+ let source;+ if (values.incremental) {+ // For native queries, use checkpoint-filter (plain string)+ // For MBQL/Python queries, use checkpoint-filter-unique-key (prefixed format)+ const strategyFields = values.checkpointFilter+ ? { "checkpoint-filter": values.checkpointFilter }+ : values.checkpointFilterUniqueKey+ ? { "checkpoint-filter-unique-key": values.checkpointFilterUniqueKey }+ : {};++ source = {+ ...transform.source,+ "source-incremental-strategy": {+ type: "checkpoint" as const,+ ...strategyFields,+ },+ };+ } else {+ source = {+ ...transform.source,+ "source-incremental-strategy": undefined,+ };+ }++ // Build the target with incremental strategy if enabled+ const target = values.incremental+ ? {+ type: "table-incremental" as const,+ name: transform.target.name,+ schema: transform.target.schema,+ database: transform.target.database,+ "target-incremental-strategy": {+ type: values.targetStrategy,+ },+ }+ : {+ type: "table" as const,+ name: transform.target.name,+ schema: transform.target.schema,+ database: transform.target.database,+ };++ try {+ await updateTransform({+ id: transform.id,+ source,+ target,+ }).unwrap();++ onUpdate();+ } catch (error) {+ sendToast({+ message: getErrorMessage(error, t`Failed to update transform`),+ icon: "warning",+ });+ }+ };++ return (+ <FormProvider+ initialValues={initialValues}+ validationSchema={validationSchema}+ onSubmit={handleSubmit}+ >+ <Form>+ <Stack gap="lg" mt="sm">+ <IncrementalTransformSettings+ source={transform.source}+ checkOnMount+ />+ <Group>+ <Box flex={1}>+ <FormErrorMessage />+ </Box>+ <Button variant="subtle" onClick={onClose}>{t`Cancel`}</Button>+ <FormSubmitButton label={t`Save`} variant="filled" />+ </Group>+ </Stack>+ </Form>+ </FormProvider>+ );+}++function getInitialValues(transform: Transform): IncrementalValues {+ const isIncremental = transform.target.type === "table-incremental";+ const strategy = transform.source["source-incremental-strategy"];++ // Read both fields from the strategy, whichever is present+ const checkpointFilter =+ strategy?.type === "checkpoint"+ ? (strategy["checkpoint-filter"] ?? null)+ : null;++ const checkpointFilterUniqueKey =+ strategy?.type === "checkpoint"+ ? (strategy["checkpoint-filter-unique-key"] ?? null)+ : null;++ return {+ incremental: isIncremental,+ sourceStrategy: "checkpoint",+ checkpointFilter: isIncremental ? checkpointFilter : null,+ checkpointFilterUniqueKey: isIncremental ? checkpointFilterUniqueKey : null,+ targetStrategy: "append",+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/utils.tsindex d9e0e657856..e8e381f5ddd 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts@@ -15,6 +15,8 @@ import type {TransformSource,} from "metabase-types/api";+import { CHECKPOINT_TEMPLATE_TAG } from "./constants";+export function parseTimestampWithTimezone(timestamp: string,systemTimezone: string | undefined,@@ -174,6 +176,8 @@ export function isNotDraftSource(return source.type !== "python" || source["source-database"] != null;}+const ALLOWED_TRANSFORM_VARIABLES = [CHECKPOINT_TEMPLATE_TAG];+export type ValidationResult = {isValid: boolean;errorMessage?: string;@@ -183,7 +187,14 @@ export function getValidationResult(query: Lib.Query): ValidationResult {const { isNative } = Lib.queryDisplayInfo(query);if (isNative) {const tags = Object.values(Lib.templateTags(query));- if (tags.some((t) => t.type !== "card" && t.type !== "snippet")) {+ // Allow snippets, cards, and the special transform variables ({checkpoint})+ const hasInvalidTags = tags.some(+ (t) =>+ t.type !== "card" &&+ t.type !== "snippet" &&+ !ALLOWED_TRANSFORM_VARIABLES.includes(t.name),+ );+ if (hasInvalidTags) {return {isValid: false,errorMessage: t`In transforms, you can use snippets and question or model references, but not variables.`,diff --git a/frontend/src/metabase-lib/metadata.ts b/frontend/src/metabase-lib/metadata.tsindex 03fdeb642ca..055748fb616 100644--- a/frontend/src/metabase-lib/metadata.ts+++ b/frontend/src/metabase-lib/metadata.ts@@ -230,6 +230,10 @@ export function columnKey(column: ColumnMetadata): string {return ML.column_key(column);}+export function columnUniqueKey(column: ColumnMetadata): string {+ return ML.column_unique_key(column);+}+export function isColumnMetadata(arg: unknown): arg is ColumnMetadata {return ML.column_metadata_QMARK_(arg);}diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex c5651f120ed..27312d6fdd1 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -39,12 +39,28 @@ export function createMockPythonTransformSource({export function createMockTransformTarget(opts?: Partial<TransformTarget>,): TransformTarget {- return {- type: "table",+ const base = {+ type: "table" as const,name: "Table",schema: null,database: 1,+ };++ if (opts?.type === "table-incremental") {+ return {+ ...base,+ ...opts,+ type: "table-incremental",+ "target-incremental-strategy": opts["target-incremental-strategy"] ?? {+ type: "append",+ },+ };+ }++ return {+ ...base,...opts,+ type: "table",};}diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex ace4df4ff1f..86a5788f813 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -29,6 +29,16 @@ export type SuggestedTransform = Partial<Pick<Transform, "id">> &export type PythonTransformTableAliases = Record<string, ConcreteTableId>;+export type TransformSourceCheckpointStrategy = {+ type: "checkpoint";+ // For native queries+ "checkpoint-filter"?: string;+ // For MBQL and Python queries+ "checkpoint-filter-unique-key"?: string;+};++export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;+export type PythonTransformSourceDraft = {type: "python";body: string;@@ -41,15 +51,20 @@ export type PythonTransformSource = {body: string;"source-database": DatabaseId;"source-tables": PythonTransformTableAliases;+ "source-incremental-strategy"?: SourceIncrementalStrategy;};export type QueryTransformSource = {type: "query";query: DatasetQuery;+ "source-incremental-strategy"?: SourceIncrementalStrategy;};export type TransformSource = QueryTransformSource | PythonTransformSource;+export type TransformTargetAppendStrategy = {+ type: "append";+};export type DraftTransformSource =| Transform["source"]| PythonTransformSourceDraft;@@ -58,15 +73,27 @@ export type DraftTransform = Partial<Pick<Transform, "id" | "name" | "description" | "target">> & { source: DraftTransformSource };-export type TransformTargetType = "table";+export type TargetIncrementalStrategy = TransformTargetAppendStrategy;++export type TransformTargetType = "table" | "table-incremental";++export type TableTarget = {+ type: "table";+ name: string;+ schema: string | null;+ database: number;+};-export type TransformTarget = {- type: TransformTargetType;+export type TableIncrementalTarget = {+ type: "table-incremental";name: string;schema: string | null;database: number;+ "target-incremental-strategy": TargetIncrementalStrategy;};+export type TransformTarget = TableTarget | TableIncrementalTarget;+export type TransformRun = {id: TransformRunId;status: TransformRunStatus;@@ -212,3 +239,18 @@ export type UpdatePythonLibraryRequest = {path: string;source: string;};++export type ExtractColumnsFromQueryRequest = {+ query: DatasetQuery;+};++export type ExtractColumnsFromQueryResponse = {+ columns: string[];+};++export type CheckQueryComplexityRequest = string;++export type CheckQueryComplexityResponse = {+ is_simple: boolean;+ reason: string;+};diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 9e210d7b80d..38c9032920d 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -878,8 +878,17 @@(defmethod driver/compile-transform :bigquery-cloud-sdk[_driver {:keys [query output-table]}]- (let [table-str (get-table-str output-table)]- [(format "CREATE OR REPLACE TABLE %s AS %s" table-str query)]))+ (let [{sql-query :query sql-params :params} query+ table-str (get-table-str output-table)]+ [(format "CREATE OR REPLACE TABLE %s AS %s" table-str sql-query)+ sql-params]))++(defmethod driver/compile-insert :bigquery-cloud-sdk+ [_driver {:keys [query output-table]}]+ (let [{sql-query :query sql-params :params} query+ table-str (get-table-str output-table)]+ [(format "INSERT INTO %s %s" table-str sql-query)+ sql-params]))(defmethod driver/compile-drop-table :bigquery-cloud-sdk[_driver table]@@ -927,7 +936,7 @@(u.date/format (u.date/parse value)):type/DateTime- (u.date/format (u.date/parse value))+ (u.date/format :iso-local-date-time (u.date/parse value)):type/DateTimeWithLocalTZ(u.date/format (u.date/parse value))@@ -974,9 +983,10 @@(try(doall(for [query queries]- (let [sql (if (string? query) query (first query))+ (let [[sql params] (if (string? query) [query] query)_ (log/debugf "Executing BigQuery DDL: %s" sql)job-config (-> (QueryJobConfiguration/newBuilder sql)+ (bigquery.params/set-parameters! params)(.setUseLegacySql false)(.build))table-result (.query client job-config (into-array BigQuery$JobOption []))]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex 3e5131efe1a..69217c4ba8b 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1346,3 +1346,14 @@:type/JSON [[:raw "JSON"]]:type/SerializedJSON [[:raw "JSON"]]:type/Decimal [[:raw "BIGDECIMAL"]])))++(deftest ^:parallel compile-transform-test+ (mt/test-driver :bigquery-cloud-sdk+ (testing "compile-transform creates CREATE OR REPLACE TABLE"+ (is (= ["CREATE OR REPLACE TABLE `PRODUCTS_COPY` AS SELECT * FROM products" nil]+ (driver/compile-transform :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}+ :output-table :PRODUCTS_COPY}))))+ (testing "compile-insert generates INSERT INTO"+ (is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil]+ (driver/compile-insert :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}+ :output-table :PRODUCTS_COPY}))))))diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex 98bd56ab095..9cf62520555 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -320,14 +320,21 @@(defmethod driver/compile-transform :clickhouse[driver {:keys [query output-table]}]- (let [pieces [(sql.qp/format-honeysql driver {:create-table output-table})+ (let [{sql-query :query sql-params :params} query+ pieces [(sql.qp/format-honeysql driver {:create-table output-table});; TODO(rileythomp, 2025-08-22): Is there a better way to do this?;; i.e. only do this if we don't have a non-nullable field to use as a primary key?(sql.qp/format-honeysql driver {:raw "ORDER BY ()"})["AS"]- (sql.qp/format-honeysql driver {:raw query})]- query (str/join " " (map first pieces))]- (into [query] (mapcat rest) pieces)))+ [sql-query sql-params]]+ sql (str/join " " (map first pieces))]+ (into [sql] (mapcat rest) pieces)))++(defmethod driver/compile-insert :clickhouse+ [driver {:keys [query output-table]}]+ (let [{sql-query :query sql-params :params} query]+ [(first (sql.qp/format-honeysql driver {:insert-into [output-table {:raw sql-query}]}))+ sql-params]))(defmethod driver/create-schema-if-needed! :clickhouse[driver conn-spec schema]diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 7166c2adce0..0e442dbb75c 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -316,10 +316,14 @@(deftest ^:parallel compile-transform-test(mt/test-driver :clickhouse- (testing "compile transform for clickhouse with empty primary key column"- (is (= ["CREATE TABLE `PRODUCTS_COPY` ORDER BY () AS SELECT * FROM products"]- (driver/compile-transform :clickhouse {:query "SELECT * FROM products"- :output-table "PRODUCTS_COPY"}))))))+ (testing "compile-transform for clickhouse with empty primary key column"+ (is (= ["CREATE TABLE `PRODUCTS_COPY` ORDER BY () AS SELECT * FROM products" nil]+ (driver/compile-transform :clickhouse {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "compile-insert generates INSERT INTO"+ (is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil]+ (driver/compile-insert :clickhouse {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))))(deftest ^:parallel clickhouse-db-supports-schemas-test(doseq [[schemas-supported? details] [[false? {}]diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex bc28c143513..5560e5f56b4 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -1024,11 +1024,18 @@(defmethod driver/compile-transform :sqlserver[driver {:keys [query output-table]}]- (let [^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))- ^Select parsed-query (macaw/parsed-query query)+ (let [{sql-query :query sql-params :params} query+ ^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))+ ^Select parsed-query (macaw/parsed-query sql-query)^PlainSelect select-body (.getSelectBody parsed-query)](.setIntoTables select-body [(Table. table-name)])- [(str parsed-query)]))+ [(str parsed-query) sql-params]))++(defmethod driver/compile-insert :sqlserver+ [driver {:keys [query output-table]}]+ (let [{sql-query :query sql-params :params} query+ ^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))]+ [(format "INSERT INTO %s %s" table-name sql-query) sql-params]))(defmethod driver/table-exists? :sqlserver[driver database {:keys [schema name] :as _table}]diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex e25915fcc8f..715ffe4ca67 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -771,3 +771,14 @@:type/Text [:text]:type/Time [:time]:type/UUID [:uniqueidentifier])))++(deftest ^:parallel compile-transform-test+ (mt/test-driver :sqlserver+ (testing "compile-transform creates SELECT INTO"+ (is (= ["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]+ (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "compile-insert generates INSERT INTO"+ (is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil]+ (driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))))diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 249a4a63863..0d94f5c74e5 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -7,7 +7,7 @@SQL-based drivers can use the `:sql` driver as a parent, and JDBC-based SQL drivers can use `:sql-jdbc`. Both ofthese drivers define additional multimethods that child drivers should implement; see [[metabase.driver.sql]] and[[metabase.driver.sql-jdbc]] for more details."- (:refer-clojure :exclude [some mapv])+ (:refer-clojure :exclude [some mapv empty?])#_{:clj-kondo/ignore [:metabase/modules]}(:require[clojure.java.io :as io]@@ -23,7 +23,7 @@[metabase.util.json :as json][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [mapv]]+ [metabase.util.performance :refer [mapv empty?]][potemkin :as p]))(set! *warn-on-reflection* true)@@ -1167,11 +1167,17 @@:hierarchy #'hierarchy)(defmulti compile-transform- "Compiles the sql for a transform statement, given an inner sql query and a destination."+ "Compiles the sql for a transform statement (CREATE TABLE AS), given a compiled inner sql query and a destination."{:added "0.57.0", :arglists '([driver {:keys [query output-table]}])}dispatch-on-initialized-driver:hierarchy #'hierarchy)+(defmulti compile-insert+ "Compiles the sql for an insert statement (INSERT INTO ... SELECT), given a compiled inner sql query and a destination."+ {:added "0.58.0", :arglists '([driver {:keys [query output-table]}])}+ dispatch-on-initialized-driver+ :hierarchy #'hierarchy)+(defmulti compile-drop-table"Compiles the sql for a drop table statement for a given table."{:added "0.57.0", :arglists '([driver table])}@@ -1211,6 +1217,10 @@[(dispatch-on-initialized-driver driver) (:type transform-details)]):hierarchy #'hierarchy)+(defmethod drop-transform-target! [::driver :table-incremental]+ [driver database target]+ ((get-method drop-transform-target! [driver :table]) driver database target))+(mr/def ::native-query-deps.table-dep[:map{:closed true}@@ -1453,14 +1463,14 @@(defmethod insert-from-source! [::driver :jsonl-file][driver db-id {:keys [columns] :as table-definition} {:keys [file]}](with-open [rdr (io/reader file)]- (let [lines (line-seq rdr)+ (let [lines (line-seq rdr)data-rows (map (fn [line](let [m (json/decode line)](mapv (fn [column](let [raw-val (get m (:name column))](insert-col->val driver :jsonl-file column raw-val)))columns)))- lines)]+ (filter (comp not empty?) lines))](insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))(defmulti add-columns!diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex 0e4ae807fef..7eacf0d78b9 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -25,6 +25,7 @@[metabase.driver.sql.query-processor.util :as sql.qp.u][metabase.driver.sql.util :as sql.u][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [deferred-tru]][metabase.util.log :as log]@@ -966,8 +967,16 @@(defmethod driver/insert-col->val [:mysql :jsonl-file][_driver _ column-def v]- (if (and (string? v) (isa? (:type column-def) :type/DateTimeWithTZ))- (t/offset-date-time v)+ (if (string? v)+ (cond+ (isa? (:type column-def) :type/DateTimeWithTZ)+ (t/offset-date-time v)++ (isa? (:type column-def) :type/DateTime)+ (u.date/parse v)++ :else+ v)v))(defn- parse-grantdiff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex c359d708ee8..31e00212173 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -127,6 +127,14 @@overwrite? (cons (driver/compile-drop-table driver output-table)))]{:rows-affected (last (driver/execute-raw-queries! driver conn-spec queries))}))+(defmethod driver/run-transform! [:sql :table-incremental]+ [driver {:keys [conn-spec database output-table] :as transform-details} _opts]+ (let [queries (if (driver/table-exists? driver database {:schema (namespace output-table)+ :name (name output-table)})+ (driver/compile-insert driver transform-details)+ (driver/compile-transform driver transform-details))]+ {:rows-affected (last (driver/execute-raw-queries! driver conn-spec [queries]))}))+(defn qualified-name"Return the name of the target table of a transform as a possibly qualified symbol."[{schema :schema, table-name :name}]diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex c6d6fb67003..5da66ba79b6 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2067,9 +2067,18 @@(defmethod driver/compile-transform :sql[driver {:keys [query output-table]}]- (format-honeysql driver- {:create-table-as [(keyword output-table)]- :raw query}))+ (let [{sql-query :query sql-params :params} query]+ [(first (format-honeysql driver+ {:create-table-as [(keyword output-table)]+ :raw sql-query}))+ sql-params]))++(defmethod driver/compile-insert :sql+ [driver {:keys [query output-table]}]+ (let [{sql-query :query sql-params :params} query]+ [(first (format-honeysql driver+ {:insert-into [(keyword output-table) {:raw sql-query}]}))+ sql-params]))(defmethod driver/compile-drop-table :sql[driver table]diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljsindex 9a831900c31..e1e02706cea 100644--- a/src/metabase/lib/js.cljs+++ b/src/metabase/lib/js.cljs@@ -72,6 +72,7 @@[metabase.lib.js.metadata :as js.metadata][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation]+ [metabase.lib.metadata.column :as lib.metadata.column][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.normalize :as lib.normalize][metabase.lib.order-by :as lib.order-by]@@ -1494,6 +1495,14 @@lib.convert/->legacy-MBQLnormalize-legacy-ref))+(defn ^:export column-unique-key+ "Given a column metadata from eg. [[fieldable-columns]], return an opaque unique key for it.+ This key is stable based on the column's alias and can be used to identify columns across operations.++ > **Code health:** Healthy. Prefer this over [[ref]] for identifying columns in UI state."+ [column]+ (lib.metadata.column/column-unique-key column))+(defn ^:export legacy-ref"Given a column, metric or segment metadata from eg. [[fieldable-columns]] or [[available-segments]],return it as a legacy JSON field ref.diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.cljindex d476c472ea1..bd215cbb072 100644--- a/test/metabase/driver/sql/transforms_test.clj+++ b/test/metabase/driver/sql/transforms_test.clj@@ -11,7 +11,7 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(testing "simple table name"(let [result (driver/compile-transform driver/*driver*- {:query "SELECT * FROM products"+ {:query {:query "SELECT * FROM products"}:output-table :my_table:primary-key "id"})](testing "returns a vector"@@ -28,7 +28,7 @@(testing "schema-qualified table name"(let [result (driver/compile-transform driver/*driver*- {:query "SELECT * FROM products"+ {:query {:query "SELECT * FROM products"}:output-table :my_schema/my_table:primary-key "id"})](testing "returns a vector"@@ -82,7 +82,7 @@(testing "execute-transform! should pass correct format to execute-raw-queries!"(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(let [compile-result (driver/compile-transform driver/*driver*- {:query "SELECT * FROM products"+ {:query {:query "SELECT * FROM products"}:output-table :my_table:primary-key "id"})drop-result (driver/compile-drop-table driver/*driver* :my_table)]@@ -126,3 +126,32 @@;; Drivers might quote these differently, but both parts should be present(is (or (re-find #"schema.*my_table" (first result))(re-find #"my_table" (first result)))))))))++(deftest compile-insert-test+ (testing "compile-insert generates INSERT INTO statements"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "simple table name"+ (let [result (driver/compile-insert driver/*driver*+ {:query {:query "SELECT * FROM products"}+ :output-table :my_table})]+ (testing "returns a vector"+ (is (vector? result)))+ (testing "first element is SQL string"+ (is (string? (first result))))+ (testing "generates INSERT INTO statement"+ (is (re-find #"(?i)INSERT\s+INTO.*my_table" (first result))))+ (testing "includes SELECT statement"+ (is (re-find #"(?i)SELECT" (first result))))))++ (testing "schema-qualified table"+ (let [result (driver/compile-insert driver/*driver*+ {:query {:query "SELECT * FROM products"}+ :output-table :my_schema/my_table})]+ (testing "returns a vector"+ (is (vector? result)))+ (testing "generates INSERT INTO statement"+ (is (re-find #"(?i)INSERT\s+INTO" (first result))))+ (testing "includes both schema and table parts"+ (let [sql (first result)]+ (is (re-find #"my_schema" sql) "Schema name should be present")+ (is (re-find #"my_table" sql) "Table name should be present"))))))))