The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Query pagination for MCP (#70971)
Noah Mossf3a2cfab7fb9
diff --git a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.cljindex dbbf7aea5d6..81ccd5448fa 100644--- a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj@@ -327,12 +327,12 @@(is (= (mt/id) (lib/database-id decoded)))(is (= (mt/id :orders) (lib/primary-source-table-id decoded))))))- (testing "Applies default limit of 100 when no limit is specified"+ (testing "Applies default limit of 200 when no limit is specified"(let [table-id (mt/id :orders)response (agent-client :rasta :post 200 "agent/v1/construct-query"{:table_id table-id})decoded (decode-query response)]- (is (= 100 (lib/current-limit decoded)))))+ (is (= 200 (lib/current-limit decoded)))))(testing "Respects explicit limit"(let [table-id (mt/id :orders)@@ -356,19 +356,25 @@:limit 5});; Streaming response returns 202 (accepted) since it starts streaming before completionexecute-resp (agent-client :rasta :post 202 "agent/v1/execute"- {:query (:query construct-resp)})- cols (get-in execute-resp [:data :cols])- rows (get-in execute-resp [:data :rows])]- (testing "response structure"- (is (= "completed" (:status execute-resp)))- (is (= 5 (:row_count execute-resp)))- (is (sequential? cols))- (is (sequential? rows))- (is (= 5 (count rows))))- (testing "column metadata"- (is (seq cols) "Should have column metadata")- (is (every? :name cols) "Each column should have a name")- (is (every? :base_type cols) "Each column should have a base_type"))))))+ {:query (:query construct-resp)})]+ (is (=? {:status "completed"+ :row_count 5+ :data {:cols (fn [cols]+ (and (seq cols)+ (every? :name cols)+ (every? :base_type cols)))+ :rows (fn [rows] (= 5 (count rows)))}}+ execute-resp))))++ (testing "Enforces agent query row limit even when query specifies a higher limit"+ (let [table-id (mt/id :orders)+ construct-resp (agent-client :rasta :post 200 "agent/v1/construct-query"+ {:table_id table-id+ :limit 300})+ execute-resp (agent-client :rasta :post 202 "agent/v1/execute"+ {:query (:query construct-resp)})]+ (is (=? {:status "completed" :row_count 200}+ execute-resp))))))(deftest execute-query-records-query-execution-test(with-agent-api-setup!@@ -383,7 +389,7 @@{:request-options {:headers headers}}{:query (:query construct-resp)});; QueryExecution is saved asynchronously, so poll for it- query-execution (tu/poll-until 2000+ query-execution (tu/poll-until 5000(t2/select-one :model/QueryExecution :executor_id user-id))](is (= :agent (:context query-execution))))))))@@ -541,6 +547,96 @@(client/client :get 403 (str "agent/v1/table/" table-id){:request-options {:headers headers}}))))))))+(deftest combined-query-test+ (with-agent-api-setup!+ (testing "Returns results for a table query"+ (let [table-id (mt/id :orders)+ field-id (visible-field-id table-id "ID")+ response (agent-client :rasta :post 202 "agent/v1/query"+ {:table_id table-id+ :order_by [{:field {:field_id field-id} :direction "asc"}]+ :limit 5})]+ (is (=? {:status "completed"+ :row_count 5+ :continuation_token string?+ :data {:cols sequential?+ :rows (fn [rows] (= 5 (count rows)))}}+ response))))++ (testing "Continuation token returns next page of results"+ (let [table-id (mt/id :orders)+ field-id (visible-field-id table-id "ID")+ page1 (agent-client :rasta :post 202 "agent/v1/query"+ {:table_id table-id+ :order_by [{:field {:field_id field-id} :direction "asc"}]+ :limit 5})+ page2 (agent-client :rasta :post 202 "agent/v1/query"+ {:continuation_token (:continuation_token page1)})]+ (is (=? {:row_count 5+ :continuation_token string?+ :data {:rows (fn [rows] (= 5 (count rows)))}}+ page1))+ (is (=? {:row_count 5+ :continuation_token string?+ :data {:rows (fn [rows] (= 5 (count rows)))}}+ page2))+ (is (not= (get-in page1 [:data :rows])+ (get-in page2 [:data :rows]))+ "Pages should return different rows")))++ (testing "No continuation_token when all rows are returned"+ (is (=? {:status "completed"+ :continuation_token nil?}+ (agent-client :rasta :post 202 "agent/v1/query"+ {:table_id (mt/id :orders)+ :aggregations [{:function "count"}]}))))++ (testing "Constraint cap limits results to 200 rows"+ (is (=? {:status "completed"+ :row_count (fn [n] (<= n 200))}+ (agent-client :rasta :post 202 "agent/v1/query"+ {:table_id (mt/id :orders)+ :limit 1000}))))))++(deftest combined-query-metric-test+ (with-agent-api-setup!+ (mt/with-temp [:model/Card metric {:name "Test Metric"+ :type :metric+ :database_id (mt/id)+ :dataset_query (orders-count-query)}]+ (testing "Returns results for a metric query"+ (is (=? {:status "completed"+ :row_count pos?}+ (agent-client :rasta :post 202 "agent/v1/query"+ {:metric_id (:id metric)})))))))++(deftest combined-query-scope-test+ (with-agent-api-setup!+ (testing "agent:query scope grants access to combined query endpoint"+ (let [headers {"authorization" (str "Bearer " (sign-jwt {:email "[email protected]"+ :scope "agent:query"}))}]+ (is (= "completed"+ (:status (client/client :post 202 "agent/v1/query"+ {:request-options {:headers headers}}+ {:table_id (mt/id :orders) :limit 1}))))))++ (testing "agent:* wildcard scope grants access"+ (let [headers {"authorization" (str "Bearer " (sign-jwt {:email "[email protected]"+ :scope "agent:*"}))}]+ (is (= "completed"+ (:status (client/client :post 202 "agent/v1/query"+ {:request-options {:headers headers}}+ {:table_id (mt/id :orders) :limit 1}))))))++ (testing "agent:query:construct scope does NOT grant access to combined endpoint"+ (let [headers {"authorization" (str "Bearer " (sign-jwt {:email "[email protected]"+ :scope "agent:query:construct"}))}]+ (is (= {:error "unsupported_scope"+ :message "Token does not have required scope: agent:query"}+ (client/client :post 403 "agent/v1/query"+ {:request-options {:headers headers}}+ {:table_id (mt/id :orders) :limit 1})))))))+(deftest search-finds-metrics-test(with-agent-api-setup!(binding [search.ingestion/*force-sync* true]diff --git a/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj b/enterprise/backend/test/metabase_enterprise/sso/test_setup.cljindex 3d74d39c795..216a0dc3dad 100644--- a/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj+++ b/enterprise/backend/test/metabase_enterprise/sso/test_setup.clj@@ -5,6 +5,7 @@to avoid circular dependencies between test namespaces."(:require[clojure.string :as str]+ [metabase.config.core :as config][metabase.premium-features.token-check :as token-check][metabase.request.core :as request][metabase.server.instance :as server.instance]@@ -92,6 +93,16 @@"Default JWT secret for tests. Note: this is regenerated on each test run."(u.random/secure-hex 32))+(defn- localhost-site-url+ "Return a valid localhost site URL for tests, even when no Jetty server is running.+ The CLI test runner often executes without a live server instance, so `server-port`+ can be nil in that mode."+ []+ (str "http://localhost:"+ (or (server.instance/server-port)+ (config/config-str :mb-jetty-port)+ 3000)))+(defn call-with-default-jwt-config!"Execute `f` with default JWT configuration set up."[f]@@ -101,7 +112,7 @@[jwt-enabled truejwt-identity-provider-uri default-jwt-idp-urijwt-shared-secret default-jwt-secret- site-url (format "http://localhost:%s" (server.instance/server-port))]+ site-url (localhost-site-url)](mt/with-premium-features current-features(f))))))@@ -137,7 +148,7 @@slack-connect-client-secret default-slack-client-secretslack-connect-authentication-mode "sso"slack-connect-user-provisioning-enabled true- site-url (format "http://localhost:%s" (server.instance/server-port))]+ site-url (localhost-site-url)](mt/with-premium-features current-features(f))))))@@ -159,7 +170,7 @@(mt/with-additional-premium-features #{:sso-oidc}(mt/with-temporary-setting-values[oidc-providers [default-oidc-provider]- site-url (format "http://localhost:%s" (server.instance/server-port))]+ site-url (localhost-site-url)](mt/with-premium-features current-features(f))))))diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex 7be34499f61..c01f29f11aa 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -34,7 +34,12 @@(def ^:private ^:const default-query-row-limit"Default row limit for table queries when no limit is specified."- 100)+ 200)++(def ^:private ^:const max-query-row-limit+ "Hard cap on rows returned by the combined query endpoint, keeping result sets lean for LLM context windows.+ Agents can paginate via continuation tokens for more."+ 200);;; ---------------------------------------------------- Helpers ------------------------------------------------------@@ -526,30 +531,42 @@"Request schema for /v1/construct-query. Accepts either table_id or metric_id."[:or ::construct-query-table-request ::construct-query-metric-request])+(mr/def ::query-request+ "Request schema for /v1/query. Accepts construct params (table_id or metric_id) or a continuation_token."+ [:multi {:dispatch (fn [m]+ (cond+ (:continuation_token m) :continuation+ (:metric_id m) :metric+ :else :table))}+ [:continuation [:map [:continuation_token ms/NonBlankString]]]+ [:table ::construct-query-table-request]+ [:metric ::construct-query-metric-request]])+(mr/def ::construct-query-response"Response containing a base64-encoded MBQL query for use with /v1/execute."[:map[:query ms/NonBlankString]])+(defn- construct-query*+ "Shared query construction: encodes args, calls the appropriate tool fn, returns the raw lib query."+ [body]+ (if (:table_id body)+ (let [args (mc/encode ::construct-query-table-request body deftool/request-transformer)]+ (:query (check-tool-result (metabot-filters/query-datasource args))))+ (let [args (mc/encode ::construct-query-metric-request body deftool/request-transformer)]+ (:query (check-tool-result (metabot-filters/query-metric args))))))+(defn- construct-table-query"Build a query from a table using the provided query components."[body]- (let [body (cond-> body- (not (:limit body)) (assoc :limit default-query-row-limit))- args (mc/encode ::construct-query-table-request body deftool/request-transformer)- data (check-tool-result (metabot-filters/query-datasource args))]- {:query (-> (:query data)- json/encode- u/encode-base64)}))+ (let [body (cond-> body (not (:limit body)) (assoc :limit default-query-row-limit))+ query (construct-query* body)]+ {:query (-> query json/encode u/encode-base64)}))(defn- construct-metric-query"Build a query from a metric using filters and group_by."[body]- (let [args (mc/encode ::construct-query-metric-request body deftool/request-transformer)- data (check-tool-result (metabot-filters/query-metric args))]- {:query (-> (:query data)- json/encode- u/encode-base64)}))+ {:query (-> (construct-query* body) json/encode u/encode-base64)})(api.macros/defendpoint :post "/v1/construct-query" :- ::construct-query-response"Construct an MBQL query from a table or metric.@@ -569,6 +586,85 @@(construct-table-query body)(construct-metric-query body)))+;;; ------------------------------------------------- Combined Query -------------------------------------------------++(defn- generate-continuation-token+ "Build a base64-encoded continuation token containing the query and next-page pagination info."+ [query-map limit page]+ (-> {:query query-map+ :pagination {:limit limit :page (inc page)}}+ json/encode+ u/encode-base64))++(defn- decode-continuation-token+ "Decode a base64-encoded continuation token into {:query ... :pagination ...}."+ [token]+ (-> token u/decode-base64 json/decode+kw))++(defn- build-query-for-execution+ "Construct a pMBQL query map from table_id or metric_id params. Returns {:query <map> :limit <int>}.+ The JSON round-trip strips lib metadata so the query is a plain pMBQL map suitable for token serialization."+ [body]+ (let [limit (min (or (:limit body) default-query-row-limit) max-query-row-limit)+ query (construct-query* (assoc body :limit limit))]+ {:query (json/decode+kw (json/encode query)) :limit limit}))++(defn- apply-page-to-query+ "Apply :page clause to the last stage of a pMBQL query map."+ [query-map page items]+ (let [stages (:stages query-map)+ last-idx (dec (count stages))]+ (assoc-in query-map [:stages last-idx :page] {:page page :items items})))++(defn- prepare-agent-query+ "Apply standard Agent API query preparation: middleware defaults and execution info."+ [query]+ (-> query+ (update-in [:middleware :js-int-to-string?] (fnil identity true))+ qp/userland-query-with-default-constraints+ (update :info merge {:executed-by api/*current-user-id*+ :context :agent})))++(defn- prepare-combined-query+ "Apply the tighter row cap used by the combined query endpoint."+ [query]+ (assoc (prepare-agent-query query)+ :constraints {:max-results max-query-row-limit+ :max-results-bare-rows max-query-row-limit}))++(api.macros/defendpoint :post "/v1/query"+ :- (streaming-response/streaming-response-schema ::query-response)+ "Query a Metabase table or metric, or continue paginating a previous query.++ Accepts either construct params (table_id/metric_id + filters, aggregations, etc.)+ or a continuation_token from a previous response. Returns results with column metadata+ and an optional continuation_token for fetching the next page."+ {:scope "agent:query"+ :tool {:name "query"+ :description (str "Query a Metabase table or metric. Returns results with column metadata. "+ "If more rows are available, the response includes a continuation_token — "+ "pass it back to get the next page.")}}+ [_route-params+ _query-params+ body :- ::query-request]+ (let [{:keys [query limit page]}+ (if-let [token (:continuation_token body)]+ (let [{:keys [query pagination]} (decode-continuation-token token)]+ {:query query :limit (:limit pagination) :page (:page pagination)})+ (let [{:keys [query limit]} (build-query-for-execution body)]+ {:query query :limit limit :page 1}))+ pmbql-with-page (apply-page-to-query query page limit)]+ (qp.streaming/streaming-response+ [rff :api]+ (qp/process-query+ (prepare-combined-query pmbql-with-page)+ (qp.streaming/transforming-query-response+ rff+ (fn [result]+ (assoc result :continuation_token+ (when (= (:row_count result) limit)+ (generate-continuation-token query limit page)))))))))+;;; ------------------------------------------------- Execute Query --------------------------------------------------(mr/def ::execute-query-request@@ -598,6 +694,11 @@[:running_time {:optional true} :int][:error {:optional true} :string]])+(mr/def ::query-response+ "Extends ::execute-query-response with an optional continuation_token for pagination."+ [:merge ::execute-query-response+ [:map [:continuation_token {:optional true} [:maybe :string]]]])+(api.macros/defendpoint :post "/v1/execute":- (streaming-response/streaming-response-schema ::execute-query-response)"Execute an MBQL query and return results.@@ -609,7 +710,7 @@- On success: {:data {:cols [...] :rows [...]} :row_count N :status :completed :running_time M}- On failure: {:status :failed :error \"message\" ...}- Standard userspace query limits are enforced (2000 rows for simple queries, 10000 for aggregated)."+ Agent query row limits are enforced (200 rows per request)."{:scope "agent:query:execute":tool {:name "execute_query":description "Execute a previously constructed query and return the results with column metadata, row count, and execution time."}}@@ -618,16 +719,9 @@{encoded-query :query} :- ::execute-query-request](let [query (-> encoded-queryu/decode-base64- json/decode+kw)- info {:executed-by api/*current-user-id*- :context :agent}]+ json/decode+kw)](qp.streaming/streaming-response [rff :api]- (qp/process-query- (-> query- (update-in [:middleware :js-int-to-string?] (fnil identity true))- qp/userland-query-with-default-constraints- (update :info merge info))- rff))))+ (qp/process-query (prepare-combined-query query) rff))));;; ------------------------------------------------- Authentication -------------------------------------------------;;diff --git a/src/metabase/mcp/tools.clj b/src/metabase/mcp/tools.cljindex 4a230847e49..e6f1cc4371d 100644--- a/src/metabase/mcp/tools.clj+++ b/src/metabase/mcp/tools.clj@@ -20,8 +20,6 @@(set! *warn-on-reflection* true)-(comment streaming-response/keep-me) ; ensure StreamingResponse class is loaded-(defn- generate-manifest"Generate tools manifest from agent API endpoint metadata."[]@@ -107,17 +105,19 @@(seq used) (update :inputSchema assoc :$defs used))))))tools)))-(defn- build-tool-index []- (into {} (map (juxt :name identity)) (:tools (manifest))))+(defn- build-tool-index+ "Build name->tool lookup from manifest tools."+ [tools]+ (into {} (map (juxt :name identity)) tools))(def ^:private tool-index-delay- (delay (build-tool-index)))+ (delay (build-tool-index (:tools @manifest-delay))))(defn- tool-index"Lookup from tool name to its full manifest definition. Cached in prod, recomputed each call in dev."[](if config/is-dev?- (build-tool-index)+ (build-tool-index (:tools (manifest)))@tool-index-delay));;; ------------------------------------------------- Tool Dispatch -------------------------------------------------@@ -132,24 +132,24 @@[message]{:content [{:type "text" :text message}] :isError true})+(comment streaming-response/keep-me) ; ensure StreamingResponse class is loaded+(defn- capture-streaming-response"Execute a StreamingResponse in-process by writing to a ByteArrayOutputStream,then parse the JSON output and return it as MCP text content.- NOTE: This intentionally bypasses the normal StreamingResponse lifecycle (thread pool,- gzip, donechan delivery) because the agent-api streaming functions only write JSON to- the output stream and don't depend on `streaming-response/*response*` or other- infrastructure bindings. BOT-1120 tracks streaming MCP responses without buffering- in memory, which will replace this approach."+ This buffers the full response in memory (~120KB for 200 rows), which is fine for+ the row limits we enforce."[^StreamingResponse response]- (let [baos (ByteArrayOutputStream.)+ (let [baos (ByteArrayOutputStream.)canceled-chan (a/promise-chan)- f (.f response)]+ f (.f response)](f baos canceled-chan)(text-content (json/decode+kw (.toString baos "UTF-8")))))(defn- deliver-agent-api-response- "Dispatch to agent API routes and deliver response to promise."+ "Dispatch to agent API routes and deliver response to promise.+ Materializes StreamingResponse bodies in-process before delivering."[result method path token-scopes body](agent-api/routes(cond-> {:request-method method@@ -157,7 +157,10 @@:metabase-user-id api/*current-user-id*:token-scopes token-scopes}body (assoc :body body))- (fn [response] (deliver result response))+ (fn [{resp-body :body :as response}]+ (deliver result (if (instance? StreamingResponse resp-body)+ (capture-streaming-response resp-body)+ response)))(fn [error] (deliver result {:status 500 :body {:message (ex-message error)}}))))(defn- invoke-agent-api@@ -171,8 +174,9 @@(deliver-agent-api-response result method path token-scopes body)(let [response (deref result 30000 {:status 504 :body {:message "Timeout"}})](cond- (instance? StreamingResponse (:body response))- (capture-streaming-response (:body response))+ ;; Already materialized from a StreamingResponse+ (:content response)+ response(= 200 (:status response))(text-content (:body response))@@ -232,7 +236,7 @@"Dispatch an MCP `tools/call` request to the appropriate handler.`token-scopes` from the original MCP session are propagated to the syntheticagent-api request so that scope restrictions are preserved.- Returns MCP content on success, or error content on failure."+ Returns MCP content maps (text-content on success, error-content on failure)."[token-scopes tool-name arguments](if-let [tool-def (get (tool-index) tool-name)](if-not (scope-matches? token-scopes (:scope tool-def))diff --git a/test/metabase/mcp/api_test.clj b/test/metabase/mcp/api_test.cljindex 8881edf1405..c262bee376f 100644--- a/test/metabase/mcp/api_test.clj+++ b/test/metabase/mcp/api_test.clj@@ -149,7 +149,7 @@(is (= 200 (:status post-response))))))(deftest tools-list-test- (testing "tools/list returns the 7 agent tools"+ (testing "tools/list returns the 8 agent tools"(let [[session-id _] (initialize!)response (mcp-request (jsonrpc-request "tools/list"){"mcp-session-id" session-id})@@ -157,7 +157,7 @@(is (= 200 (:status response)))(is (pos? (count tools)))(is (= #{"search" "get_table" "get_metric" "get_table_field_values"- "get_metric_field_values" "construct_query" "execute_query"}+ "get_metric_field_values" "construct_query" "execute_query" "query"}(set (map :name tools))))(testing "each tool has a description and inputSchema"(doseq [tool tools]@@ -409,7 +409,7 @@(deftest tools-list-scope-filtering-test(testing "tools/list with unrestricted scopes returns all tools"(let [tools (mcp.tools/list-tools #{::scope/unrestricted})]- (is (= 7 (count tools)))))+ (is (= 8 (count tools)))))(testing "tools/list with specific scope only returns matching tools"(let [tools (mcp.tools/list-tools #{"agent:search"})@@ -422,11 +422,11 @@(testing "tools/list with wildcard scope matches all agent tools"(let [tools (mcp.tools/list-tools #{"agent:*"})]- (is (= 7 (count tools)))))+ (is (= 8 (count tools)))))(testing "tools/list with nil scopes returns all tools"(let [tools (mcp.tools/list-tools nil)]- (is (= 7 (count tools)))))+ (is (= 8 (count tools)))))(testing "tools/list with empty scopes does not return all tools"(let [tools (mcp.tools/list-tools #{})]