The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Convert legacy into mbql5 on API level (#71944)
lbrdnka0a787ea5408
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5bacdc1a01c..bfbb7b05e0d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1195,6 +1195,7 @@documentsdriverinternal-stats+ legacy-mbqlliblib-bellmdiff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex a25bda6a537..6866a2956bc 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -3,12 +3,15 @@(:require[clojure.core.async :as a][clojure.string :as str]+ [medley.core :as m][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers][metabase.app-db.core :as app-db][metabase.config.core :as config]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.metabot.agent.core :as agent][metabase.metabot.api.document][metabase.metabot.api.metabot]@@ -27,6 +30,7 @@[metabase.slackbot.api][metabase.util :as u][metabase.util.log :as log]+ [metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2])(:import@@ -208,6 +212,28 @@:state state:debug? debug?})))+(defn- legacy->modern-query+ [query]+ (if-not (= :mbql-version/legacy (lib/normalized-mbql-version query))+ query+ (lib/query+ (lib-be/application-database-metadata-provider (:database query))+ query)))++(def upgradable-item-types+ "User is viewing item types with query and chart configs. Upgradeable by [[upgrade-viewing-queries]]."+ metabot.context/item-types-qc)++(mu/defn- upgrade-viewing-queries+ "Update queries of items in viewing context vector. Handles following item types: adhoc, question, model, metric"+ [viewing :- [:vector metabot.context/ViewingItemSchema]]+ (letfn [(update-items-query [item] (m/update-existing item :query legacy->modern-query))+ (maybe-update-item [item] (cond-> item+ (contains? upgradable-item-types (:type item))+ (-> update-items-query+ (m/update-existing :chart_configs (partial mapv update-items-query)))))]+ (mapv maybe-update-item viewing)))+;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen;;@@ -229,7 +255,8 @@[:chart-configs {:optional true} [:map-of :string :any]]]][:debug {:optional true} [:maybe :boolean]]]](metabot.context/log body :llm.log/fe->be)- (streaming-request body))+ (let [body* (m/update-existing body [:context :user_is_viewing] upgrade-viewing-queries)]+ (streaming-request body*)));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happendiff --git a/src/metabase/metabot/context.clj b/src/metabase/metabot/context.cljindex 7ef6a830889..b17bdde67c9 100644--- a/src/metabase/metabot/context.clj+++ b/src/metabase/metabot/context.clj@@ -1,10 +1,12 @@(ns metabase.metabot.context(:require[clojure.java.io :as io]+ [malli.core][medley.core :as m][metabase.activity-feed.core :as activity-feed][metabase.api.common :as api][metabase.config.core :as config]+ ^{:clj-kondo/ignore [:discouraged-namespace :metabase/modules]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]@@ -46,34 +48,63 @@(.newLine w)(.newLine w)))))-(def item-types- "Allowed values for the `:type` key of `:user_is_viewing` item."+(def item-types-qc+ "Item types types storing query and chart configs."#{"adhoc"-"question"- "model""metric"+ "model"})- "document"- "dashboard"- "transform"- "code_editor"})+(def item-types+ "Allowed values for the `:type` key of `:user_is_viewing` item."+ (into item-types-qc+ #{"document"+ "dashboard"+ "transform"+ "code_editor"}))(def ^:private item-type-schema"Schema for the `:type` key of `:user_is_viewing` item."(into [:enum] item-types))+(def DefaultItemSchema+ "Default schema of viewing context item."+ [:map+ [:type item-type-schema]+ [:query+ {:optional true}+ [:or+ ::lib.schema/query+ ::mbql.s/Query]]])++(def QcItemSchema+ "Schema viewing context item with query and charts."+ [:map+ [:type (into [:enum] item-types-qc)]+ [:query+ {:optional true}+ [:or+ ::lib.schema/query+ ::mbql.s/Query]]+ [:chart_configs+ {:optional true}+ [:vector+ [:map+ [:query+ {:optional true}+ [:or+ ::lib.schema/query+ ::mbql.s/Query]]]]]])++(def ViewingItemSchema+ "Schema of user is viewing item."+ [:or QcItemSchema DefaultItemSchema])+(mr/def ::context[:and[:map-of :keyword :any][:map- [:user_is_viewing- {:optional true}- [:vector [:map- [:type item-type-schema]- [:query- {:optional true}- ::lib.schema/query]]]]]])+ [:user_is_viewing {:optional true} [:vector ViewingItemSchema]]]])(defn- query-for-sql-parsing"Given an item in context, return the query if it is a native query or SQL transform that can have table usage parseddiff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 30ca539228c..e183567bb23 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -6,6 +6,10 @@[compojure.response][medley.core :as m][metabase.config.core :as config]+ [metabase.lib.convert :as lib.convert]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta][metabase.llm.settings :as llm.settings][metabase.metabot.api :as api][metabase.metabot.config :as metabot.config]@@ -493,3 +497,75 @@(is (= [{:type :text, :text "solo"}](into [] (#'api/combine-text-parts-xf)[{:type :text, :text "solo"}])))))++(defn- legacy-query+ "A legacy inner-query-style map suitable for [[#'api/upgrade-viewing-queries]]."+ []+ {:database (mt/id)+ :query {:source-table (mt/id :orders)}+ :type :query})++(deftest upgrade-viewing-queries-upgradable-types-test+ (doseq [item-type ["adhoc" "question" "metric" "model"]]+ (testing (str "upgrades query for type=" item-type)+ (let [result (#'api/upgrade-viewing-queries [{:type item-type :query (legacy-query)}])+ q (:query (first result))]+ (is (= :mbql/query (:lib/type q)))+ (is (= (mt/id) (:database q)))))))++(deftest upgrade-viewing-queries-chart-configs-test+ (let [lq (legacy-query)+ item {:type "adhoc"+ :query lq+ :chart_configs [{:query lq}+ {:query lq}]}+ result (first (#'api/upgrade-viewing-queries [item]))]+ (is (= :mbql/query (:lib/type (:query result))))+ (is (every? #(= :mbql/query (:lib/type (:query %)))+ (:chart_configs result)))))++(deftest upgrade-viewing-queries-missing-keys-test+ (testing "items without :query are unchanged"+ (let [item {:type "adhoc"}]+ (is (= [item] (#'api/upgrade-viewing-queries [item])))))+ (testing "items without :chart_configs keep no chart_configs"+ (let [result (first (#'api/upgrade-viewing-queries [{:type "question" :query (legacy-query)}]))]+ (is (nil? (:chart_configs result))))))++(deftest upgrade-viewing-queries-mixed-items-test+ (let [lq (legacy-query)+ items [{:type "adhoc" :query lq}+ {:type "dashboard"}+ {:type "model" :query lq :chart_configs [{:query lq}]}]+ result (#'api/upgrade-viewing-queries items)]+ (is (=? [{:query {:lib/type :mbql/query}}+ {}+ {:query {:lib/type :mbql/query}+ :chart_configs [{:query {:lib/type :mbql/query}}]}]+ result))))++(deftest upgrade-viewing-queries-idempotence-test+ (let [mp meta/metadata-provider+ q (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ items [{:type "adhoc" :query q}+ {:type "dashboard"}+ {:type "model" :query q :chart_configs [{:query q}]}]+ result (#'api/upgrade-viewing-queries items)]+ (is (=? [{:type "adhoc" :query q}+ {:type "dashboard"}+ {:type "model" :query q :chart_configs [{:query q}]}]+ result))))++(deftest ^:parallel upgrade-viewing-queries-native-test+ (testing "Native queries are properly adjusted"+ (let [mp (mt/metadata-provider)+ native (lib/native-query mp "select * from orders")+ legacy (lib.convert/->legacy-MBQL native)+ items [{:type "adhoc" :query legacy}+ {:type "dashboard"}+ {:type "model" :query legacy :chart_configs [{:query legacy}]}]+ result (#'api/upgrade-viewing-queries items)]+ (is (=? [{:type "adhoc" :query native}+ {:type "dashboard"}+ {:type "model" :query native :chart_configs [{:query native}]}]+ result)))))