Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert transforms changes recently (#68446)

dpsutton410471a4d024
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index b6fed336758..e100e92e66d 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -20,7 +20,6 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
- [metabase.lib.core :as lib]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.schema :as qp.schema]
[metabase.request.core :as request]
@@ -123,83 +122,6 @@
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
(get-transforms query-params))
-(defn- extract-all-columns-from-query
- "Extracts column metadata (name and type) from a query.
-
- Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.
-
- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
- to inspect the query structure. This works for most modern JDBC drivers but may not
- be supported by all drivers or for all query types."
- [driver database-id query]
- (try
- (let [{:keys [query]} (qp.compile/compile query)]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database-id
- {}
- (fn [conn]
- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
- (when-let [rsmeta (.getMetaData stmt)]
- (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))
- (catch Exception e
- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
- nil)))
-
-(defn- extract-incremental-filter-columns-from-query
- "Extracts column names suitable for incremental transform checkpoint filtering.
-
- This function is specifically for incremental transform checkpoint column selection.
- It only returns columns with types supported for checkpoint filtering:
- - Temporal types (timestamp, timestamp with timezone)
- - Numeric types (integer, float, decimal)
-
- Text, boolean, and other types are filtered out as they are not supported for
- incremental checkpointing.
-
- Returns a vector of column names (as strings), or nil if extraction fails.
-
- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
- to inspect the query structure. This works for most modern JDBC drivers but may not
- be supported by all drivers or for all query types."
- [driver database-id query]
- (some->> (extract-all-columns-from-query driver database-id query)
- (filter (comp transforms.util/supported-incremental-filter-type? :base_type))
- (mapv :name)))
-
-(defn- validate-incremental-column-type!
- "Validates that the checkpoint column for an incremental transform has a supported type.
-
- For MBQL/Python transforms, resolves the column from the query using the unique key.
- For native queries, extracts columns from the query and checks the checkpoint-filter column.
-
- Throws a 400 error if the column type is not supported or cannot be resolved."
- [{:keys [source]}]
- (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}
- (:source-incremental-strategy source)]
- (when (= "checkpoint" strategy-type)
- (let [{:keys [query]} source
- database-id (:database query)
- database (api/check-404 (t2/select-one :model/Database :id database-id))
- driver-name (driver/the-initialized-driver (:engine database))]
- (cond
- ;; For MBQL/Python with unique key, resolve column from query metadata
- checkpoint-filter-unique-key
- (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]
- (api/check-400 column (deferred-tru "Checkpoint column not found in query."))
- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))
- (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."
- (pr-str (:base-type column)))))
-
- ;; For native query with checkpoint-filter, validate type if we can extract the column metadata
- checkpoint-filter
- (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]
- (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]
- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))
- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
- checkpoint-filter
- (pr-str (:base_type column)))))))))))
-
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
@@ -221,7 +143,6 @@
(api/check-superuser)
(check-database-feature body)
(check-feature-enabled! body)
- (validate-incremental-column-type! body)
(api/check (not (transforms.util/target-table-exists? body))
403
@@ -337,7 +258,6 @@
;; we must validate on a full transform object
(check-feature-enabled! new)
(check-database-feature new)
- (validate-incremental-column-type! new)
(when (transforms.util/query-transform? old)
(when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
(throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
@@ -423,6 +343,30 @@
:run_id run-id})
(assoc :status 202)))))
+(defn- extract-columns-from-query
+ "Attempts to extract column names from an MBQL query without executing it.
+
+ Returns a vector of column names (as strings), or nil if extraction fails.
+
+ The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()
+ to inspect the query structure. This works for most modern JDBC drivers but may not
+ be supported by all drivers or for all query types."
+ [driver database-id query]
+ (try
+ (let [{:keys [query]} (qp.compile/compile query)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database-id
+ {}
+ (fn [conn]
+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
+ (when-let [rsmeta (.getMetaData stmt)]
+ (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]
+ (seq (mapv :name columns))))))))
+ (catch Exception e
+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
+ nil)))
+
(defn- simple-native-query?
"Checks if a native SQL query string is simple enough for automatic checkpoint insertion."
[sql-string]
@@ -463,19 +407,16 @@
(api/check-superuser)
(simple-native-query? query))
+;; TODO (Cam 2025-12-04) please add a response schema to this API endpoint, it makes it easier for our customers to
+;; use our API + we will need it when we make auto-TypeScript-signature generation happen
+;;
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/extract-columns"
- :- [:map [:columns [:maybe [:sequential :string]]]]
- "Extract column names suitable for incremental transform checkpoint filtering.
-
- This endpoint is specifically for populating the checkpoint column dropdown in
- incremental transforms. It only returns columns with types supported for checkpoint
- filtering: temporal (timestamp/tz) and numeric (int/float) types.
-
- Text, boolean, and other unsupported column types are filtered out.
+ "Extract column names from an MBQL query without executing it.
- The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],
+ Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],
which handles parameterized queries with template tags. Then extracts column names
- and types using PreparedStatement metadata.
+ using PreparedStatement metadata.
Returns a map with a :columns key containing a vector of column names (strings).
If extraction fails, returns nil for :columns."
@@ -485,9 +426,9 @@
[:query ::qp.schema/any-query]]]
(api/check-superuser)
(let [database-id (:database query)
- database (api/check-404 (t2/select-one :model/Database :id database-id))
+ database (api/check-404 (t2/select-one :model/Database :id database-id))
driver-name (driver/the-initialized-driver (:engine database))
- columns (extract-incremental-filter-columns-from-query driver-name database-id query)]
+ columns (extract-columns-from-query driver-name database-id query)]
{:columns columns}))
(def ^{:arglists '([request respond raise])} routes
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index fabbb72b355..0891687d0fa 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -214,14 +214,6 @@
[source]
(= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
-(defn supported-incremental-filter-type?
- "Returns true if the given base-type is supported for incremental filtering.
-
- We only support temporal (timestamp/tz) and numeric (int/float) types."
- [base-type]
- (or (isa? base-type :type/Temporal)
- (isa? base-type :type/Number)))
-
(defn- source->checkpoint-filter-unique-key
"Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
[query source-incremental-strategy]
@@ -231,29 +223,15 @@
"Resolve the checkpoint filter column for an incremental transform.
Tries to resolve the column using the unique key first.
- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.
-
- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).
- Throws an exception if the column type is not supported."
+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."
[query source-incremental-strategy table metadata-provider]
- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]
- (when-some [{column-name :name
- :keys [base-type]
- :as column}
- (cond
- checkpoint-filter-unique-key
- (source->checkpoint-filter-unique-key query source-incremental-strategy)
- checkpoint-filter
- (when-some [field-id (t2/select-one-pk :model/Field
- :table_id (:id table)
- :name checkpoint-filter)]
- (lib.metadata/field metadata-provider field-id)))]
- (when-not (supported-incremental-filter-type? base-type)
- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "
- "Only numeric and temporal columns are supported for incremental filtering.")
- {:column-name column-name
- :base-type base-type})))
- column)))
+ (or
+ (source->checkpoint-filter-unique-key query source-incremental-strategy)
+ (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]
+ (when-let [field-id (t2/select-one-pk :model/Field
+ :table_id (:id table)
+ :name field-name)]
+ (lib.metadata/field metadata-provider field-id)))))
(defn next-checkpoint
"Build a query to compute the MAX of the checkpoint column from the target table.
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index c346cae39e7..a7d95408a1d 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -3,7 +3,6 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
- [metabase-enterprise.transforms.api]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :refer [get-test-schema
@@ -1033,8 +1032,7 @@
(testing "Successfully extracts columns from a simple SELECT query"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
{:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]
- (is (= ["id" "price"] (:columns response))
- "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))
+ (is (= ["id" "name" "category" "price"] (:columns response)))))
(testing "Returns nil for invalid SQL"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
@@ -1044,14 +1042,7 @@
(testing "Extracts columns from query with aliases"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
{:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]
- (is (= ["product_id"] (:columns response))
- "Should only return numeric column (id), filtering out text column (name)")))
-
- (testing "Filters columns by type - only returns numeric and temporal columns"
- (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
- {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]
- (is (= ["id" "price" "created_at"] (:columns response))
- "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))
+ (is (= ["product_id" "product_name"] (:columns response)))))
(testing "Requires superuser permissions"
(is (= "You don't have permissions to do that."
@@ -1184,102 +1175,3 @@
(let [items (:data (mt/user-http-request :rasta :get 200
(format "collection/%d/items" collection-id)))]
(is (empty? items))))))))
-
-(deftest native-incremental-column-type-validated-on-create-test
- (testing "POST /api/ee/transform column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/with-premium-features #{:transforms}
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (testing "Rejects unsupported checkpoint column type (text)"
- (let [response (mt/user-http-request :crowberto :post 400 "ee/transform"
- {:name "Invalid Incremental Transform"
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, category, price FROM transforms_products")
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "name"}}
- :target {:type "table-incremental"
- :schema schema
- :name "invalid_incremental"
- :target-incremental-strategy {:type "append"}}})]
- (is (string? response))
- (is (re-find #"Only numeric and temporal" response))))))))))
-
-(deftest native-incremental-column-type-validated-on-update-test
- (testing "PUT /api/ee/transform column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/with-premium-features #{:transforms}
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (with-transform-cleanup! [table-name "update_incremental_test"]
- (let [;; Create a non-incremental transform first
- created (mt/user-http-request :crowberto :post 200 "ee/transform"
- {:name "Test Transform"
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, category FROM transforms_products")}
- :target {:type "table"
- :schema schema
- :name table-name}})]
- (testing "Rejects update to unsupported checkpoint column type (text)"
- (let [response (mt/user-http-request :crowberto :put 400
- (format "ee/transform/%d" (:id created))
- {:source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, category FROM transforms_products")
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "category"}}
- :target {:type "table-incremental"
- :schema schema
- :name table-name
- :target-incremental-strategy {:type "append"}}})]
- (is (string? response))
- (is (re-find #"Only numeric and temporal" response))))))))))))
-
-(deftest mbql-incremental-column-type-validated-on-create-test
- (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/with-premium-features #{:transforms}
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (testing "Rejects unsupported checkpoint column type (text)"
- (let [query (mt/mbql-query transforms_products)
- response (mt/user-http-request :crowberto :post 400 "ee/transform"
- {:name "Invalid MBQL Incremental"
- :source {:type "query"
- :query query
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter-unique-key "column-unique-key-v1$name"}}
- :target {:type "table-incremental"
- :schema schema
- :name "invalid_mbql_incremental"
- :target-incremental-strategy {:type "append"}}})]
- (is (string? response))
- (is (re-find #"not supported" response))))))))))
-
-(deftest native-incremental-column-validation-when-not-extractable-test
- (testing "Native query checkpoint column validation with text input fallback"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/with-premium-features #{:transforms}
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (testing "Accepts any column if they were not extractable"
- (with-transform-cleanup! [table-name "fallback_test_unextracted"]
- (with-redefs [metabase-enterprise.transforms.api/extract-all-columns-from-query
- ;; simulate lack of driver support for extraction
- (fn [_driver _database-id _query] nil)]
- (let [response (mt/user-http-request :crowberto :post 200 "ee/transform"
- {:name "Unextracted Column - Text Input Fallback"
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, created_at FROM transforms_products")
- :source-incremental-strategy {:type "checkpoint"
- ;; created_at is in the query but not in our stubbed extraction
- :checkpoint-filter "created_at"}}
- :target {:type "table-incremental"
- :schema schema
- :name table-name
- :target-incremental-strategy {:type "append"}}})]
- (is (some? (:id response))
- "Should accept column not in extracted metadata, allowing text input fallback")))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 8801067128c..8f32f1a2262 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -481,29 +481,6 @@
(is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")
(is (some? checkpoint) "Checkpoint should be updated")))))))))))))))
-(deftest unsupported-checkpoint-column-type-test
- (testing "Transform fails at runtime with unsupported checkpoint column type"
- (mt/test-drivers (test-drivers)
- (mt/with-premium-features #{:transforms}
- (mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "unsupported_type_test"]
- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- ;; Create transform with text column (unsupported type) as checkpoint
- transform-payload {:name "Invalid Checkpoint Type Transform"
- :source {:type "query"
- :query (make-incremental-source-query-without-template-tag schema)
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "name"}}
- :target {:type "table-incremental"
- :schema schema
- :name target-table
- :database (mt/id)
- :target-incremental-strategy {:type "append"}}}]
- (testing "API validation rejects unsupported checkpoint column type"
- (let [response (mt/user-http-request :crowberto :post 400 "ee/transform" transform-payload)]
- (is (string? response))
- (is (re-find #"not supported" response)))))))))))
-
(deftest ^:postgres-only native-query-with-temporal-checkpoint-test
(testing "Native query with temporal checkpoint"
;; we test only in postgres because it's easy to cast to ::timestamp
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 188543ba92a..9e5566e583f 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -173,7 +173,7 @@
;;; ------------------------------------------- Tests for sync edge cases --------------------------------------------
-(deftest edge-case-identifiers-test
+(deftest ^:sequential edge-case-identifiers-test
(mt/test-driver :postgres
(testing "Make sure that Tables / Fields with dots in their names get escaped properly"
(mt/dataset dots-in-names