The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Streaming API response schema for documentation (#68386)
Noah Moss110cb5bca7d1
diff --git a/src/metabase/api/macros/defendpoint/open_api.clj b/src/metabase/api/macros/defendpoint/open_api.cljindex 35966c7a45c..19649ca4925 100644--- a/src/metabase/api/macros/defendpoint/open_api.clj+++ b/src/metabase/api/macros/defendpoint/open_api.clj@@ -131,9 +131,17 @@(mu/defn- schema->response-obj :- [:maybe :metabase.api.open-api/path-item.responses]"Convert a Malli schema to an OpenAPI response schema.- This is used to convert the `:response-schema` in [[metabase.api.macros/defendpoint]] to an OpenAPI response schema."+ This is used to convert the `:response-schema` in [[metabase.api.macros/defendpoint]] to an OpenAPI response schema.++ If the schema has `:openapi/response-schema` in its properties (e.g., for streaming responses), that schema+ is used for documentation instead of the actual schema. This allows streaming endpoints to document the+ JSON content they return while validating that the return value is a StreamingResponse instance."[schema]- (let [jss-schema (mjs-collect-definitions schema)]+ (let [resolved-schema (mr/resolve-schema schema)+ ;; Check for :openapi/response-schema in the schema properties - used by server/streaming-response-schema+ content-schema (or (-> resolved-schema mc/properties :openapi/response-schema)+ schema)+ jss-schema (mjs-collect-definitions content-schema)]{"2XX" (-> {:description (or (:description jss-schema) "Successful response")}(assoc :content {"application/json" {:schema (fix-json-schema jss-schema)}}))}))diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 0c0ef46df0d..fb3f0ee8d60 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -26,6 +26,7 @@[metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.schema :as qp.schema][metabase.query-processor.streaming :as qp.streaming]+ [metabase.server.core :as server][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]@@ -88,11 +89,8 @@rff))(qp/process-query (update query :info merge info) rff))))))-;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to-;; use our API + we will need it when we make auto-TypeScript-signature generation happen-;;-#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/"+ :- (server/streaming-response-schema ::qp.schema/query-result)"Execute a query and retrieve the results in the usual format. The query will not use the cache."[_route-params_query-params@@ -123,11 +121,8 @@json-key(keyword json-key)))-;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to-;; use our API + we will need it when we make auto-TypeScript-signature generation happen-;;-#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post ["/:export-format", :export-format qp.schema/export-formats-regex]+ :- (server/streaming-response-schema ::qp.schema/query-result)"Execute a query and download the result data as a file in the specified format."[{:keys [export-format]} :- [:map[:export-format ::qp.schema/export-format]]@@ -212,11 +207,8 @@(cond-> compiledpretty (update :query prettify)))))-;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to-;; use our API + we will need it when we make auto-TypeScript-signature generation happen-;;-#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/pivot"+ :- (server/streaming-response-schema ::qp.schema/query-result)"Generate a pivoted dataset for an ad-hoc query"[_route-params_query-paramsdiff --git a/src/metabase/query_processor/schema.clj b/src/metabase/query_processor/schema.cljindex 237c31aa299..2a5ed813b46 100644--- a/src/metabase/query_processor/schema.clj+++ b/src/metabase/query_processor/schema.clj@@ -80,3 +80,35 @@(mr/def ::result-metadata.columns"A sequence of result metadata columns as returned by the Query Processor."[:sequential ::result-metadata.column])++;;; ------------------------------------------------ Query Results -------------------------------------------------++(mr/def ::query-result.data+ "Schema for the :data key of query results."+ [:map+ [:cols [:sequential ::result-metadata.column]]+ [:rows [:sequential [:sequential :any]]]+ [:native_form {:optional true} :map]+ [:results_timezone {:optional true} :string]+ [:results_metadata {:optional true} [:map+ [:columns [:sequential ::result-metadata.column]]]]+ [:insights {:optional true} [:sequential :map]]+ [:download_perms {:optional true} :string]+ [:is_sandboxed {:optional true} :boolean]+ [:format-rows? {:optional true} :boolean]])++(mr/def ::query-result+ "Schema for query execution results returned by the Query Processor."+ [:map+ [:status [:enum :completed :failed]]+ [:row_count :int]+ [:data {:optional true} ::query-result.data]+ [:running_time {:optional true} :int]+ [:started_at {:optional true} :string]+ [:database_id {:optional true} ::lib.schema.id/database]+ [:json_query {:optional true} :map]+ [:average_execution_time {:optional true} [:maybe :int]]+ [:context {:optional true} :any]+ [:cached {:optional true} [:maybe :string]]+ [:error {:optional true} :string]+ [:error_type {:optional true} :keyword]])diff --git a/src/metabase/server/core.clj b/src/metabase/server/core.cljindex 0ff7d80ccad..5ef84ce99ba 100644--- a/src/metabase/server/core.clj+++ b/src/metabase/server/core.clj@@ -8,13 +8,15 @@[metabase.server.middleware.json][metabase.server.protocols][metabase.server.routes]+ [metabase.server.streaming-response][potemkin :as p]))(commentmetabase.server.handler/keep-memetabase.server.instance/keep-memetabase.server.protocols/keep-me- metabase.server.routes/keep-me)+ metabase.server.routes/keep-me+ metabase.server.streaming-response/keep-me)(p/import-vars[metabase.server.handler@@ -30,4 +32,6 @@make-routes][metabase.server.middleware.jsonwrap-json-body- wrap-streamed-json-response])+ wrap-streamed-json-response]+ [metabase.server.streaming-response+ streaming-response-schema])diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 2b1ec2982a7..012b2d46574 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -321,3 +321,26 @@{:pre [(= (count bindings) 2)]}`(-streaming-response (bound-fn [~(vary-meta os-binding assoc :tag 'java.io.OutputStream) ~canceled-chan-binding] ~@body)~options))++;;;; Malli schema for StreamingResponse++(defn streaming-response-schema+ "Malli schema for a streaming HTTP response that will contain JSON matching `content-schema`.++ At runtime, validates that the response is a StreamingResponse instance.+ WARNING: DOES NOT VALIDATE the actual data being streamed at runtime. For OpenAPI documentation, uses `content-schema`+ to describe the JSON response body.++ Example:+ (api.macros/defendpoint :post \"/query\"+ :- (server/streaming-response-schema+ [:map+ [:data [:map [:cols sequential?] [:rows sequential?]]]+ [:row_count :int]])+ ...)"+ [content-schema]+ [:fn+ {:openapi/response-schema content-schema+ :description "Streaming JSON response"+ :error/message "Non-streaming response returned from streaming endpoint"}+ #(instance? StreamingResponse %)])diff --git a/test/metabase/api/macros/defendpoint/open_api_test.clj b/test/metabase/api/macros/defendpoint/open_api_test.cljindex ee472635418..d19de7fb943 100644--- a/test/metabase/api/macros/defendpoint/open_api_test.clj+++ b/test/metabase/api/macros/defendpoint/open_api_test.clj@@ -5,6 +5,7 @@[malli.json-schema :as mjs][metabase.api.macros.defendpoint.open-api :as defendpoint.open-api][metabase.lib.schema.common :as lib.schema.common]+ [metabase.server.core :as server][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -80,3 +81,21 @@:body []}result (#'defendpoint.open-api/path-item "/api/upload" form)](is (true? (:deprecated result)))))))++(deftest ^:parallel streaming-response-schema-test+ (testing "server/streaming-response-schema uses content schema for OpenAPI docs"+ (binding [defendpoint.open-api/*definitions* (atom (sorted-map))]+ (let [content-schema [:map+ [:data [:map [:cols [:sequential :any]] [:rows [:sequential :any]]]]+ [:row_count :int]+ [:status [:enum "completed" "failed"]]]+ response-schema (server/streaming-response-schema content-schema)+ result (#'defendpoint.open-api/schema->response-obj response-schema)]+ (testing "generates 2XX response"+ (is (contains? result "2XX")))+ (testing "uses content schema properties, not StreamingResponse fn"+ (let [json-schema (get-in result ["2XX" :content "application/json" :schema])]+ (is (= :object (:type json-schema)))+ (is (contains? (:properties json-schema) "data"))+ (is (contains? (:properties json-schema) "row_count"))+ (is (contains? (:properties json-schema) "status"))))))))diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 21e6f89beb6..8f3bf247085 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -4,6 +4,7 @@[clojure.core.async :as a][clojure.test :refer :all][compojure.response]+ [malli.error :as me][metabase.driver :as driver][metabase.query-processor.pipeline :as qp.pipeline][metabase.server.instance :as server.instance]@@ -13,7 +14,8 @@[metabase.test :as mt][metabase.test.http-client :as client][metabase.util :as u]- [metabase.util.json :as json])+ [metabase.util.json :as json]+ [metabase.util.malli.registry :as mr])(:import(jakarta.servlet AsyncContext ServletOutputStream)(jakarta.servlet.http HttpServletResponse)@@ -330,3 +332,14 @@"Response should not contain :via key")(is (= "preserve-me" (:custom-data error-response))"Response should still include custom data")))))))++(deftest ^:parallel streaming-response-schema-error-test+ (testing "streaming-response-schema correctly validates responses"+ (let [schema (streaming-response/streaming-response-schema [:map [:data :any]])]+ (testing "StreamingResponse instances pass validation"+ (is (mr/validate schema+ (streaming-response/-streaming-response (fn [_ _]) {}))))+ (testing "Non-StreamingResponse values fail with a clear error"+ (is (not (mr/validate schema {:data "not a streaming response"})))+ (is (= ["Non-streaming response returned from streaming endpoint"]+ (me/humanize (mr/explain schema {:data "not a streaming response"}))))))))