The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Ad hoc query API for workspaces (#69274)
Chris Truter3286107b159c
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex 4a1067dd6c0..660dd3aa8ad 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -33,6 +33,7 @@[metabase.util.i18n :as i18n :refer [deferred-tru tru]][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]+ [metabase.util.secret :as u.secret][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -60,7 +61,10 @@[:database_id ::ws.t/appdb-id][:status ::status][:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:updated_at ms/TemporalInstant]+ ;; Only present in POST response - the unmasked API key for workspace service user.+ ;; This is the only time the key is available; after creation it's hashed and unrecoverable.+ [:api_key {:optional true} :string]]);; Transform-related schemas (adapted from transforms/api.clj);; TODO (Chris 2026-02-02) -- We should reuse these schemas, by exposing common types from the transforms module. They *can* match exactly.@@ -141,7 +145,8 @@"/transform/validate/target$""/run$""/transform/[^/]+/run$"- "/transform/[^/]+/dry-run$"]+ "/transform/[^/]+/dry-run$"+ "/query$"]:put ["/transform/[^/]+$"]:delete ["/transform/[^/]+$"]}(partial mapv ws-pattern)))@@ -177,11 +182,13 @@(routes.common/wrap-middleware-for-open-api-spec-generation authorize*))(defn- ws->response- "Transform a workspace record into an API response, computing the backwards-compatible status."+ "Transform a workspace record into an API response, computing the backwards-compatible status.+ If :api_key is present (only at creation time), it is preserved in the response."[ws](-> ws- (select-keys [:id :name :collection_id :database_id :created_at :updated_at])- (assoc :status (ws.model/computed-status ws))))+ (select-keys [:id :name :collection_id :database_id :created_at :updated_at :api_key])+ (assoc :status (ws.model/computed-status ws))+ (cond-> (:api_key ws) (update :api_key u.secret/expose))));;; routes@@ -848,7 +855,7 @@(defn- fetch-ws-transform [ws-id tx-id](-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)- (t2/select-one :ref_id tx-id :workspace_id ws-id)+ (t2/select-one :workspace_id ws-id :ref_id tx-id)api/check-404attach-isolated-target))@@ -869,9 +876,14 @@[:source {:optional true} ::transform-source][:target {:optional true} ::transform-target]]](t2/with-transaction [_tx]- (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))- (let [source-or-target-changed? (or (:source body) (:target body))]- (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} body)+ (let [existing (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))+ ;; Merge only :database and :schema from existing target to preserve them when not explicitly provided.+ ;; Other fields are NOT merged, allowing them to be removed by omitting from the request.+ base (select-keys (:target existing) [:database :schema])+ merged-body (cond-> body+ (:target body) (update :target #(merge base %)))+ source-or-target-changed? (or (:source body) (:target body))]+ (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body);; If source or target changed, increment versions for re-analysis(when source-or-target-changed?(ws.impl/increment-analysis-version! ws-id tx-id)@@ -887,7 +899,7 @@{:access :workspace}[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]](api/check-404 (pos? (t2/update! :model/WorkspaceTransform- {:ref_id tx-id :workspace_id ws-id}+ {:workspace_id ws-id, :ref_id tx-id}{:archived_at [:now]})));; Increment graph version since transform is leaving the graph(ws.impl/increment-graph-version! ws-id)@@ -898,7 +910,7 @@{:access :workspace}[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]](api/check-404 (pos? (t2/update! :model/WorkspaceTransform- {:ref_id tx-id :workspace_id ws-id}+ {:workspace_id ws-id, :ref_id tx-id}{:archived_at nil})));; Increment both versions - transform re-enters graph and needs re-analysis(ws.impl/increment-analysis-version! ws-id tx-id)@@ -911,7 +923,7 @@Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."{:access :workspace}[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]- (api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id)))+ (api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id)));; Increment graph version since transform is potentially leaving the graph, or reverting to the global definition.(ws.impl/increment-graph-version! ws-id)nil)@@ -930,7 +942,7 @@_query-params{:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]](let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")_ (check-transforms-enabled! (:database_id workspace))graph (ws.impl/get-or-calculate-graph! workspace)@@ -948,7 +960,7 @@run-ancestors? (assoc :ancestors ancestors-result))))(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/dry-run"- :- ::ws.t/dry-run-result+ :- ::ws.t/query-result"Dry-run a transform in a workspace without persisting to the target table.Returns the first 2000 rows of transform output for preview purposes.@@ -962,7 +974,7 @@_query-params{:keys [run_stale_ancestors]} :- [:map [:run_stale_ancestors {:optional true} ::ws.t/flag]]](let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))- transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))+ transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")_ (check-transforms-enabled! (:database_id workspace))graph (ws.impl/get-or-calculate-graph! workspace)@@ -978,6 +990,26 @@(cond-> dry-run-resultrun-ancestors? (assoc :ancestors ancestors-result))))+(api.macros/defendpoint :post "/:ws-id/query"+ :- ::ws.t/query-result+ "Execute an arbitrary SQL query in the workspace's isolated database context.+ Table references are remapped to isolated workspace tables.+ Returns the first 2000 rows of query results."+ {:access :workspace}+ [{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]+ _query-params+ {:keys [sql]} :- [:map [:sql [:string {:min 1}]]]]+ (let [workspace (api/check-404 (t2/select-one :model/Workspace ws-id))+ _ (api/check-400 (not= :archived (:base_status workspace))+ "Cannot query archived workspace")+ _ (api/check-400 (not= :uninitialized (:db_status workspace))+ "Workspace is not initialized")+ _ (api/check-400 (some? (:database_details workspace))+ "Workspace is not ready for queries")+ _ (check-transforms-enabled! (:database_id workspace))+ graph (ws.impl/get-or-calculate-graph! workspace)]+ (ws.impl/execute-adhoc-query workspace graph sql)))+(def ^:private CheckoutTransformLegacy"Legacy format for workspace checkout transforms (DEPRECATED)."[:map@@ -1019,7 +1051,7 @@:database_id db-id:base_status [:not= :archived]{:order-by [[:name :asc]]})- checkouts (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id]+ checkouts (t2/select [:model/WorkspaceTransform :workspace_id :ref_id :name]:global_id transform-id)ws-id->checkouts (into {} (map (juxt :workspace_id identity) checkouts))id->workspace (into {} (map (juxt :id identity) workspaces))]diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/common.cljindex 6fc1150c240..3a953c8d8aa 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj@@ -41,7 +41,8 @@(str stripped-name " (" next-num ")"))))(defn- create-workspace-container!- "Create the workspace and its related collection, user, and api key."+ "Create the workspace and its related collection, user, and api key.+ Returns the workspace with :api_key attached (unmasked, only available at creation time)."[creator-id db-id workspace-name](let [key-name (format "API key for Workspace: %s" workspace-name);; Ensure the key-name is unique.@@ -68,9 +69,9 @@ws (assoc ws :collection_id (:id coll))];; Set the backlink from the workspace to the collection inside it and set the schema.(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)})- ;; TODO (Sanya 2025-11-18) -- For now we expose this in logs for manual testing. In future we need a secure channel.- (log/infof "Generated API key for workspace: %s" (:unmasked_key api-key))- ws))+ ;; Return the workspace with the unmasked API key attached.+ ;; This is the only time the key is available - after this it's hashed and unrecoverable.+ (assoc ws :api_key (:unmasked_key api-key))))(defn- unique-constraint-violation?"Check if an exception is due to a unique constraint violation."diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex 7f68e176cfe..13595e2f9b5 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -127,33 +127,58 @@{:status :failed:message (:message result)})))-(defn- dry-run-sql- "Run SQL transform query and return first 2000 rows without persisting.- Returns a ::ws.t/dry-run-result map with data nested under :data to match /api/dataset format."- [{:keys [source]} remapping]+(defn- run-query+ "Remap source, execute query, and format as ::ws.t/query-result.+ Options: :row-limit (default 2000), :error-context (for logging).+ Returns {:status :succeeded :data {...} :running_time ...} or {:status :failed :message ...}."+ [source source-type remapping {:keys [row-limit error-context]+ :or {row-limit preview-row-limit}}](try- (let [s-type (transforms/transform-source-type source)- table-mapping (:tables remapping no-mapping)+ (let [table-mapping (:tables remapping no-mapping)field-mapping (:fields remapping no-mapping)- remapped-source (remap-source table-mapping field-mapping s-type source)- query (:query remapped-source)- result (qp/process-query- (assoc query :constraints {:max-results preview-row-limit- :max-results-bare-rows preview-row-limit}))+ remapped-source (remap-source table-mapping field-mapping source-type source)+ query (-> (:query remapped-source)+ (qp/userland-query)+ (assoc :constraints {:max-results row-limit+ :max-results-bare-rows row-limit}))+ result (qp/process-query query)data (:data result)](if (= :completed (:status result))- {:status :succeeded- :data (select-keys data [:rows :cols :results_metadata])}+ {:status :succeeded+ :data (select-keys data [:rows :cols :results_metadata])+ :running_time (:running_time result)+ :started_at (:started_at result)}{:status :failed:message (or (:error result) "Query execution failed")}))(catch Exception e- (log/error e "Failed to run sql dry-run")+ (log/error e error-context){:status :failed:message (ex-message e)})))+(defn- dry-run-sql+ "Run SQL transform query and return first 2000 rows without persisting.+ Returns a ::ws.t/query-result map with data nested under :data to match /api/dataset format."+ [{:keys [source]} remapping]+ (run-query source (transforms/transform-source-type source) remapping+ {:error-context "Failed to run sql dry-run"}))++(defn execute-adhoc-sql+ "Execute an arbitrary SQL query against a database and return up to row-limit rows (default 2000).+ Applies workspace table remapping so queries can reference global table names.+ Returns a ::ws.t/query-result map with data nested under :data."+ ([database-id sql remapping]+ (execute-adhoc-sql database-id sql remapping {}))+ ([database-id sql remapping opts]+ (let [source {:query {:database database-id+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/native+ :native sql}]}}]+ (run-query source :native remapping+ (merge {:error-context "Failed to execute adhoc SQL query"} opts)))))+(defn run-transform-preview"Execute transform and return first 2000 rows without persisting.- Returns a ::ws.t/dry-run-result map with data nested under :data."+ Returns a ::ws.t/query-result map with data nested under :data."[{:keys [source] :as transform} remapping](let [s-type (transforms/transform-source-type source)](case s-typediff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.cljindex ea9ff3ccf81..f8323b7d9ab 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj@@ -398,6 +398,16 @@workspace(ws.execute/run-transform-preview transform (build-remapping workspace graph))))+(defn execute-adhoc-query+ "Execute an arbitrary SQL query in the workspace's isolated database context.+ Applies workspace table remapping so queries can reference global table names.+ Options: :row-limit (default 2000).+ Returns a ::ws.t/query-result."+ [{db-id :database_id :as workspace} graph sql & {:as opts}]+ (ws.isolation/with-workspace-isolation+ workspace+ (ws.execute/execute-adhoc-sql db-id sql (build-remapping workspace graph) opts)))+;;;; ---------------------------------------- External Transform Sync ----------------------------------------(defn- extract-external-transform-idsdiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj b/enterprise/backend/src/metabase_enterprise/workspaces/types.cljindex 6ed2e5e69cf..d0a45228d7c 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/types.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/types.clj@@ -44,12 +44,14 @@[:schema {:optional true} [:maybe :string]]]][:ancestors {:optional true} ::ancestors-result]])-(mr/def ::dry-run-result- "Result of a transform dry-run (preview without persisting).+(mr/def ::query-result+ "Result of a query preview (dry-run or ad-hoc query).Data is nested under :data to match /api/dataset response format."[:map[:status [:enum :succeeded :failed]][:message {:optional true} [:maybe :string]]+ [:running_time {:optional true} [:maybe :int]]+ [:started_at {:optional true} [:maybe :any]][:data {:optional true}[:map[:rows {:optional true} [:sequential :any]]diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex 215a22349a8..d72c49d4661 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -1370,10 +1370,146 @@:not_run []}(:ancestors result))))))))))+;;; ---------------------------------------- Adhoc Query Tests ----------------------------------------++(deftest ^:synchronized adhoc-query-test+ (testing "POST /api/ee/workspace/:id/query"+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Test")]+ (testing "happy path - returns query results for valid SQL"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql "SELECT 1 as id, 'hello' as name"})]+ (is (=? {:status "succeeded"+ :data {:rows [[1 "hello"]]+ :cols [{:name #"(?i)id"} {:name #"(?i)name"}]}}+ result))))++ (testing "returns results for multi-row queries"+ (let [sql "SELECT 1 as num UNION ALL SELECT 2 UNION ALL SELECT 3 ORDER BY 1"+ result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql sql})]+ (is (=? {:status "succeeded"+ :data {:rows [[1] [2] [3]]}}+ result)))))))++(deftest ^:synchronized adhoc-query-error-handling-test+ (testing "POST /api/ee/workspace/:id/query error handling"+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Error Test")]+ (testing "returns failed status with message for invalid SQL"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql "SELECT FROM WHERE INVALID"})]+ (is (=? {:status "failed"+ :message string?}+ result))))++ (testing "returns failed status for syntax errors"+ (let [result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql "SELECT 1 LIMIT"})]+ (is (=? {:status "failed"+ :message string?}+ result)))))))++(deftest ^:synchronized adhoc-query-validation-test+ (testing "POST /api/ee/workspace/:id/query validation"+ (ws.tu/with-workspaces! [ws {:name "Adhoc Query Validation Test"}]+ (testing "returns 404 for non-existent workspace"+ (is (= "Not found."+ (mt/user-http-request :crowberto :post 404+ (ws-url 999999 "/query")+ {:sql "SELECT 1"}))))++ (testing "returns 400 for missing sql parameter"+ (is (=? {:errors {:sql "string with length >= 1"}}+ (mt/user-http-request :crowberto :post 400+ (ws-url (:id ws) "/query")+ {}))))++ (testing "returns 400 for empty sql parameter"+ (is (=? {:errors {:sql "string with length >= 1"}}+ (mt/user-http-request :crowberto :post 400+ (ws-url (:id ws) "/query")+ {:sql ""})))))))++(deftest ^:synchronized adhoc-query-archived-workspace-test+ (testing "POST /api/ee/workspace/:id/query on archived workspace"+ (let [ws (ws.tu/create-ready-ws! "Adhoc Query Archived Test")]+ ;; Archive the workspace+ (mt/user-http-request :crowberto :post 200 (ws-url (:id ws) "/archive"))++ (testing "returns 400 for archived workspace"+ (is (= "Cannot query archived workspace"+ (mt/user-http-request :crowberto :post 400+ (ws-url (:id ws) "/query")+ {:sql "SELECT 1"})))))))++(deftest ^:synchronized adhoc-query-remapping-test+ (testing "POST /api/ee/workspace/:id/query remaps table references to isolated tables"+ (ws.tu/with-workspaces! [ws {:name "Adhoc Query Remapping Test"}]+ (let [target-schema (driver.sql/default-schema driver/*driver*)+ target-table (str "adhoc_remap_" (str/replace (str (random-uuid)) "-" "_"))+ transform-def {:name "Remapping Test Transform"+ :source {:type "query"+ :query (mt/native-query+ {:query "SELECT 1 as id, 'remapped' as status"})}+ :target {:type "table"+ :database (mt/id)+ :schema target-schema+ :name target-table}}+ ;; Add transform to workspace+ ref-id (:ref_id (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/transform")+ transform-def))+ ws (ws.tu/ws-done! (:id ws))]+ ;; Run the transform to populate the isolated table+ (let [run-result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/transform/" ref-id "/run"))]+ (is (= "succeeded" (:status run-result)) "Transform should run successfully"))++ (testing "ad-hoc query can SELECT from transform output using schema-qualified table name"+ (let [query-sql (str "SELECT * FROM " target-schema "." target-table)+ result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql query-sql})]+ (is (=? {:status "succeeded"+ :data {:rows [[1 "remapped"]]+ :cols [{:name #"(?i)id"} {:name #"(?i)status"}]}}+ result))))++ (testing "ad-hoc query can SELECT from transform output using unqualified table name"+ (let [query-sql (str "SELECT * FROM " target-table)+ result (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql query-sql})]+ (is (=? {:status "succeeded"+ :data {:rows [[1 "remapped"]]+ :cols [{:name #"(?i)id"} {:name #"(?i)status"}]}}+ result))))))))++(deftest ^:synchronized adhoc-query-uses-isolated-credentials-test+ (testing "POST /api/ee/workspace/:id/query executes with workspace isolated credentials"+ (let [isolated? (atom false)+ workspace-used (atom nil)+ ws (ws.tu/create-ready-ws! "Adhoc Query Isolation Test")]+ (mt/with-dynamic-fn-redefs [ws.isolation/do-with-workspace-isolation+ (fn [workspace thunk]+ (reset! isolated? true)+ (reset! workspace-used workspace)+ (thunk))]+ (mt/user-http-request :crowberto :post 200+ (ws-url (:id ws) "/query")+ {:sql "SELECT 1"}))+ (is @isolated? "Query should execute within workspace isolation context")+ (is (= (:id ws) (:id @workspace-used)) "Should use correct workspace for isolation")+ (is (some? (:database_details @workspace-used))+ "Workspace should have database_details for proper isolation"))))+(defn- random-target [db-id]{:type "table":database db-id- :schema "transform_output"+ :schema (driver.sql/default-schema driver/*driver*):name (str/replace (str "t_" (random-uuid)) "-" "_")})(defn- my-native-query [db-id sql & [card-mapping]]@@ -1995,7 +2131,8 @@[:post "/:ws-id/run"][:post "/:ws-id/transform/:tx-id/run"][:post "/:ws-id/transform/:tx-id/dry-run"]- [:post "/:ws-id/transform/validate/target"]])+ [:post "/:ws-id/transform/validate/target"]+ [:post "/:ws-id/query"]])(def ^:private permission-denied-msg "You don't have permissions to do that.")