Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Ad hoc query API for workspaces (#69274)

Chris Truter3286107b159c
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
index 4a1067dd6c0..660dd3aa8ad 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
@@ -33,6 +33,7 @@
[metabase.util.i18n :as i18n :refer [deferred-tru tru]]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
+ [metabase.util.secret :as u.secret]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -60,7 +61,10 @@
[:database_id ::ws.t/appdb-id]
[:status ::status]
[:created_at ms/TemporalInstant]
- [:updated_at ms/TemporalInstant]])
+ [:updated_at ms/TemporalInstant]
+ ;; Only present in POST response - the unmasked API key for workspace service user.
+ ;; This is the only time the key is available; after creation it's hashed and unrecoverable.
+ [:api_key {:optional true} :string]])
;; Transform-related schemas (adapted from transforms/api.clj)
;; TODO (Chris 2026-02-02) -- We should reuse these schemas, by exposing common types from the transforms module. They *can* match exactly.
@@ -141,7 +145,8 @@
"/transform/validate/target$"
"/run$"
"/transform/[^/]+/run$"
- "/transform/[^/]+/dry-run$"]
+ "/transform/[^/]+/dry-run$"
+ "/query$"]
:put ["/transform/[^/]+$"]
:delete ["/transform/[^/]+$"]}
(partial mapv ws-pattern)))
@@ -177,11 +182,13 @@
(routes.common/wrap-middleware-for-open-api-spec-generation authorize*))
(defn- ws->response
- "Transform a workspace record into an API response, computing the backwards-compatible status."
+ "Transform a workspace record into an API response, computing the backwards-compatible status.
+ If :api_key is present (only at creation time), it is preserved in the response."
[ws]
(-> ws
- (select-keys [:id :name :collection_id :database_id :created_at :updated_at])
- (assoc :status (ws.model/computed-status ws))))
+ (select-keys [:id :name :collection_id :database_id :created_at :updated_at :api_key])
+ (assoc :status (ws.model/computed-status ws))
+ (cond-> (:api_key ws) (update :api_key u.secret/expose))))
;;; routes
@@ -848,7 +855,7 @@
(defn- fetch-ws-transform [ws-id tx-id]
(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)
- (t2/select-one :ref_id tx-id :workspace_id ws-id)
+ (t2/select-one :workspace_id ws-id :ref_id tx-id)
api/check-404
attach-isolated-target))
@@ -869,9 +876,14 @@
[:source {:optional true} ::transform-source]
[:target {:optional true} ::transform-target]]]
(t2/with-transaction [_tx]
- (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
- (let [source-or-target-changed? (or (:source body) (:target body))]
- (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} body)
+ (let [existing (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))
+ ;; Merge only :database and :schema from existing target to preserve them when not explicitly provided.
+ ;; Other fields are NOT merged, allowing them to be removed by omitting from the request.
+ base (select-keys (:target existing) [:database :schema])
+ merged-body (cond-> body
+ (:target body) (update :target #(merge base %)))
+ source-or-target-changed? (or (:source body) (:target body))]
+ (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body)
;; If source or target changed, increment versions for re-analysis
(when source-or-target-changed?
(ws.impl/increment-analysis-version! ws-id tx-id)
@@ -887,7 +899,7 @@
{:access :workspace}
[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform
- {:ref_id tx-id :workspace_id ws-id}
+ {:workspace_id ws-id, :ref_id tx-id}
{:archived_at [:now]})))
;; Increment graph version since transform is leaving the graph
(ws.impl/increment-graph-version! ws-id)
@@ -898,7 +910,7 @@
{:access :workspace}
[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform
- {:ref_id tx-id :workspace_id ws-id}
+ {:workspace_id ws-id, :ref_id tx-id}
{:archived_at nil})))
;; Increment both versions - transform re-enters graph and needs re-analysis
(ws.impl/increment-analysis-version! ws-id tx-id)
@@ -911,7 +923,7 @@
Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."
{:access :workspace}
[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
- (api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id)))
+ (api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id)))
;; Increment graph version since transform is potentially leaving the graph, or reverting to the global definition.
(ws.impl/increment-graph-version! ws-id)
nil)
@@ -930,7 +942,7 @@
_query-params
{:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]
(let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))
_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
_ (check-transforms-enabled! (:database_id workspace))
graph (ws.impl/get-or-calculate-graph! workspace)
@@ -948,7 +960,7 @@
run-ancestors? (assoc :ancestors ancestors-result))))
(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/dry-run"
- :- ::ws.t/dry-run-result
+ :- ::ws.t/query-result
"Dry-run a transform in a workspace without persisting to the target table.
Returns the first 2000 rows of transform output for preview purposes.
@@ -962,7 +974,7 @@
_query-params
{:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]]
(let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))
_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
_ (check-transforms-enabled! (:database_id workspace))
graph (ws.impl/get-or-calculate-graph! workspace)
@@ -978,6 +990,26 @@
(cond-> dry-run-result
run-ancestors? (assoc :ancestors ancestors-result))))
+(api.macros/defendpoint :post "/:ws-id/query"
+ :- ::ws.t/query-result
+ "Execute an arbitrary SQL query in the workspace's isolated database context.
+ Table references are remapped to isolated workspace tables.
+ Returns the first 2000 rows of query results."
+ {:access :workspace}
+ [{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
+ _query-params
+ {:keys [sql]} :- [:map [:sql [:string {:min 1}]]]]
+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))
+ _ (api/check-400 (not= :archived (:base_status workspace))
+ "Cannot query archived workspace")
+ _ (api/check-400 (not= :uninitialized (:db_status workspace))
+ "Workspace is not initialized")
+ _ (api/check-400 (some? (:database_details workspace))
+ "Workspace is not ready for queries")
+ _ (check-transforms-enabled! (:database_id workspace))
+ graph (ws.impl/get-or-calculate-graph! workspace)]
+ (ws.impl/execute-adhoc-query workspace graph sql)))
+
(def ^:private CheckoutTransformLegacy
"Legacy format for workspace checkout transforms (DEPRECATED)."
[:map
@@ -1019,7 +1051,7 @@
:database_id db-id
:base_status [:not= :archived]
{:order-by [[:name :asc]]})
- checkouts (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id]
+ checkouts (t2/select [:model/WorkspaceTransform :workspace_id :ref_id :name]
:global_id transform-id)
ws-id->checkouts (into {} (map (juxt :workspace_id identity) checkouts))
id->workspace (into {} (map (juxt :id identity) workspaces))]
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
index 6fc1150c240..3a953c8d8aa 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
@@ -41,7 +41,8 @@
(str stripped-name " (" next-num ")"))))
(defn- create-workspace-container!
- "Create the workspace and its related collection, user, and api key."
+ "Create the workspace and its related collection, user, and api key.
+ Returns the workspace with :api_key attached (unmasked, only available at creation time)."
[creator-id db-id workspace-name]
(let [key-name (format "API key for Workspace: %s" workspace-name)
;; Ensure the key-name is unique.
@@ -68,9 +69,9 @@
ws (assoc ws :collection_id (:id coll))]
;; Set the backlink from the workspace to the collection inside it and set the schema.
(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)})
- ;; TODO (Sanya 2025-11-18) -- For now we expose this in logs for manual testing. In future we need a secure channel.
- (log/infof "Generated API key for workspace: %s" (:unmasked_key api-key))
- ws))
+ ;; Return the workspace with the unmasked API key attached.
+ ;; This is the only time the key is available - after this it's hashed and unrecoverable.
+ (assoc ws :api_key (:unmasked_key api-key))))
(defn- unique-constraint-violation?
"Check if an exception is due to a unique constraint violation."
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index 7f68e176cfe..13595e2f9b5 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -127,33 +127,58 @@
{:status :failed
:message (:message result)})))
-(defn- dry-run-sql
- "Run SQL transform query and return first 2000 rows without persisting.
- Returns a ::ws.t/dry-run-result map with data nested under :data to match /api/dataset format."
- [{:keys [source]} remapping]
+(defn- run-query
+ "Remap source, execute query, and format as ::ws.t/query-result.
+ Options: :row-limit (default 2000), :error-context (for logging).
+ Returns {:status :succeeded :data {...} :running_time ...} or {:status :failed :message ...}."
+ [source source-type remapping {:keys [row-limit error-context]
+ :or {row-limit preview-row-limit}}]
(try
- (let [s-type (transforms/transform-source-type source)
- table-mapping (:tables remapping no-mapping)
+ (let [table-mapping (:tables remapping no-mapping)
field-mapping (:fields remapping no-mapping)
- remapped-source (remap-source table-mapping field-mapping s-type source)
- query (:query remapped-source)
- result (qp/process-query
- (assoc query :constraints {:max-results preview-row-limit
- :max-results-bare-rows preview-row-limit}))
+ remapped-source (remap-source table-mapping field-mapping source-type source)
+ query (-> (:query remapped-source)
+ (qp/userland-query)
+ (assoc :constraints {:max-results row-limit
+ :max-results-bare-rows row-limit}))
+ result (qp/process-query query)
data (:data result)]
(if (= :completed (:status result))
- {:status :succeeded
- :data (select-keys data [:rows :cols :results_metadata])}
+ {:status :succeeded
+ :data (select-keys data [:rows :cols :results_metadata])
+ :running_time (:running_time result)
+ :started_at (:started_at result)}
{:status :failed
:message (or (:error result) "Query execution failed")}))
(catch Exception e
- (log/error e "Failed to run sql dry-run")
+ (log/error e error-context)
{:status :failed
:message (ex-message e)})))
+(defn- dry-run-sql
+ "Run SQL transform query and return first 2000 rows without persisting.
+ Returns a ::ws.t/query-result map with data nested under :data to match /api/dataset format."
+ [{:keys [source]} remapping]
+ (run-query source (transforms/transform-source-type source) remapping
+ {:error-context "Failed to run sql dry-run"}))
+
+(defn execute-adhoc-sql
+ "Execute an arbitrary SQL query against a database and return up to row-limit rows (default 2000).
+ Applies workspace table remapping so queries can reference global table names.
+ Returns a ::ws.t/query-result map with data nested under :data."
+ ([database-id sql remapping]
+ (execute-adhoc-sql database-id sql remapping {}))
+ ([database-id sql remapping opts]
+ (let [source {:query {:database database-id
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/native
+ :native sql}]}}]
+ (run-query source :native remapping
+ (merge {:error-context "Failed to execute adhoc SQL query"} opts)))))
+
(defn run-transform-preview
"Execute transform and return first 2000 rows without persisting.
- Returns a ::ws.t/dry-run-result map with data nested under :data."
+ Returns a ::ws.t/query-result map with data nested under :data."
[{:keys [source] :as transform} remapping]
(let [s-type (transforms/transform-source-type source)]
(case s-type
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
index ea9ff3ccf81..f8323b7d9ab 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
@@ -398,6 +398,16 @@
workspace
(ws.execute/run-transform-preview transform (build-remapping workspace graph))))
+(defn execute-adhoc-query
+ "Execute an arbitrary SQL query in the workspace's isolated database context.
+ Applies workspace table remapping so queries can reference global table names.
+ Options: :row-limit (default 2000).
+ Returns a ::ws.t/query-result."
+ [{db-id :database_id :as workspace} graph sql & {:as opts}]
+ (ws.isolation/with-workspace-isolation
+ workspace
+ (ws.execute/execute-adhoc-sql db-id sql (build-remapping workspace graph) opts)))
+
;;;; ---------------------------------------- External Transform Sync ----------------------------------------
(defn- extract-external-transform-ids
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
index 6ed2e5e69cf..d0a45228d7c 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj
@@ -44,12 +44,14 @@
[:schema {:optional true} [:maybe :string]]]]
[:ancestors {:optional true} ::ancestors-result]])
-(mr/def ::dry-run-result
- "Result of a transform dry-run (preview without persisting).
+(mr/def ::query-result
+ "Result of a query preview (dry-run or ad-hoc query).
Data is nested under :data to match /api/dataset response format."
[:map
[:status [:enum :succeeded :failed]]
[:message {:optional true} [:maybe :string]]
+ [:running_time {:optional true} [:maybe :int]]
+ [:started_at {:optional true} [:maybe :any]]
[:data {:optional true}
[:map
[:rows {:optional true} [:sequential :any]]
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index 215a22349a8..d72c49d4661 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -1370,10 +1370,146 @@
:not_run []}
(:ancestors result))))))))))
+;;; ---------------------------------------- Adhoc Query Tests ----------------------------------------
+
+(deftest ^:synchronized adhoc-query-test
+ (testing "POST /api/ee/workspace/:id/query"
+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Test")]
+ (testing "happy path - returns query results for valid SQL"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql "SELECT 1 as id, 'hello' as name"})]
+ (is (=? {:status "succeeded"
+ :data {:rows [[1 "hello"]]
+ :cols [{:name #"(?i)id"} {:name #"(?i)name"}]}}
+ result))))
+
+ (testing "returns results for multi-row queries"
+ (let [sql "SELECT 1 as num UNION ALL SELECT 2 UNION ALL SELECT 3 ORDER BY 1"
+ result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql sql})]
+ (is (=? {:status "succeeded"
+ :data {:rows [[1] [2] [3]]}}
+ result)))))))
+
+(deftest ^:synchronized adhoc-query-error-handling-test
+ (testing "POST /api/ee/workspace/:id/query error handling"
+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Error Test")]
+ (testing "returns failed status with message for invalid SQL"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql "SELECT FROM WHERE INVALID"})]
+ (is (=? {:status "failed"
+ :message string?}
+ result))))
+
+ (testing "returns failed status for syntax errors"
+ (let [result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql "SELECT 1 LIMIT"})]
+ (is (=? {:status "failed"
+ :message string?}
+ result)))))))
+
+(deftest ^:synchronized adhoc-query-validation-test
+ (testing "POST /api/ee/workspace/:id/query validation"
+ (ws.tu/with-workspaces! [ws {:name "Adhoc Query Validation Test"}]
+ (testing "returns 404 for non-existent workspace"
+ (is (= "Not found."
+ (mt/user-http-request :crowberto :post 404
+ (ws-url 999999 "/query")
+ {:sql "SELECT 1"}))))
+
+ (testing "returns 400 for missing sql parameter"
+ (is (=? {:errors {:sql "string with length >= 1"}}
+ (mt/user-http-request :crowberto :post 400
+ (ws-url (:id ws) "/query")
+ {}))))
+
+ (testing "returns 400 for empty sql parameter"
+ (is (=? {:errors {:sql "string with length >= 1"}}
+ (mt/user-http-request :crowberto :post 400
+ (ws-url (:id ws) "/query")
+ {:sql ""})))))))
+
+(deftest ^:synchronized adhoc-query-archived-workspace-test
+ (testing "POST /api/ee/workspace/:id/query on archived workspace"
+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Archived Test")]
+ ;; Archive the workspace
+ (mt/user-http-request :crowberto :post 200 (ws-url (:id ws) "/archive"))
+
+ (testing "returns 400 for archived workspace"
+ (is (= "Cannot query archived workspace"
+ (mt/user-http-request :crowberto :post 400
+ (ws-url (:id ws) "/query")
+ {:sql "SELECT 1"})))))))
+
+(deftest ^:synchronized adhoc-query-remapping-test
+ (testing "POST /api/ee/workspace/:id/query remaps table references to isolated tables"
+ (ws.tu/with-workspaces! [ws {:name "Adhoc Query Remapping Test"}]
+ (let [target-schema (driver.sql/default-schema driver/*driver*)
+ target-table (str "adhoc_remap_" (str/replace (str (random-uuid)) "-" "_"))
+ transform-def {:name "Remapping Test Transform"
+ :source {:type "query"
+ :query (mt/native-query
+ {:query "SELECT 1 as id, 'remapped' as status"})}
+ :target {:type "table"
+ :database (mt/id)
+ :schema target-schema
+ :name target-table}}
+ ;; Add transform to workspace
+ ref-id (:ref_id (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/transform")
+ transform-def))
+ ws (ws.tu/ws-done! (:id ws))]
+ ;; Run the transform to populate the isolated table
+ (let [run-result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/transform/" ref-id "/run"))]
+ (is (= "succeeded" (:status run-result)) "Transform should run successfully"))
+
+ (testing "ad-hoc query can SELECT from transform output using schema-qualified table name"
+ (let [query-sql (str "SELECT * FROM " target-schema "." target-table)
+ result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql query-sql})]
+ (is (=? {:status "succeeded"
+ :data {:rows [[1 "remapped"]]
+ :cols [{:name #"(?i)id"} {:name #"(?i)status"}]}}
+ result))))
+
+ (testing "ad-hoc query can SELECT from transform output using unqualified table name"
+ (let [query-sql (str "SELECT * FROM " target-table)
+ result (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql query-sql})]
+ (is (=? {:status "succeeded"
+ :data {:rows [[1 "remapped"]]
+ :cols [{:name #"(?i)id"} {:name #"(?i)status"}]}}
+ result))))))))
+
+(deftest ^:synchronized adhoc-query-uses-isolated-credentials-test
+ (testing "POST /api/ee/workspace/:id/query executes with workspace isolated credentials"
+ (let [isolated? (atom false)
+ workspace-used (atom nil)
+ ws (ws.tu/create-ready-ws! "Adhoc Query Isolation Test")]
+ (mt/with-dynamic-fn-redefs [ws.isolation/do-with-workspace-isolation
+ (fn [workspace thunk]
+ (reset! isolated? true)
+ (reset! workspace-used workspace)
+ (thunk))]
+ (mt/user-http-request :crowberto :post 200
+ (ws-url (:id ws) "/query")
+ {:sql "SELECT 1"}))
+ (is @isolated? "Query should execute within workspace isolation context")
+ (is (= (:id ws) (:id @workspace-used)) "Should use correct workspace for isolation")
+ (is (some? (:database_details @workspace-used))
+ "Workspace should have database_details for proper isolation"))))
+
(defn- random-target [db-id]
{:type "table"
:database db-id
- :schema "transform_output"
+ :schema (driver.sql/default-schema driver/*driver*)
:name (str/replace (str "t_" (random-uuid)) "-" "_")})
(defn- my-native-query [db-id sql & [card-mapping]]
@@ -1995,7 +2131,8 @@
[:post "/:ws-id/run"]
[:post "/:ws-id/transform/:tx-id/run"]
[:post "/:ws-id/transform/:tx-id/dry-run"]
- [:post "/:ws-id/transform/validate/target"]])
+ [:post "/:ws-id/transform/validate/target"]
+ [:post "/:ws-id/query"]])
(def ^:private permission-denied-msg "You don't have permissions to do that.")