The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Native query context formatting (#71566)
lbrdnkbee5aa840a80
diff --git a/resources/metabot/prompts/message_injection.selmer b/resources/metabot/prompts/message_injection.selmerindex 5fe37395118..5de7c8b2e62 100644--- a/resources/metabot/prompts/message_injection.selmer+++ b/resources/metabot/prompts/message_injection.selmer@@ -4,10 +4,9 @@ Current date: {{ current_time }}In the system, the first day of the week (index 1) is: {{ first_day_of_week }}{% if current_user_info %}-<user>Here is some information about the user:{{ current_user_info|safe }}-</user>+{% endif %}{% if viewing_context %}diff --git a/src/metabase/metabot/agent/user_context.clj b/src/metabase/metabot/agent/user_context.cljindex 9341c1a8a35..78137fcdd97 100644--- a/src/metabase/metabot/agent/user_context.clj+++ b/src/metabase/metabot/agent/user_context.clj@@ -58,8 +58,7 @@[context](when-let [viewing (:user_is_viewing context)](some (fn [item]- (when (= "native" (effective-context-type item))- (some-> (:sql_engine item) u/lower-case-en)))+ (some-> item :sql_engine u/lower-case-en))viewing)));;; Context Normalization@@ -95,10 +94,7 @@Handles the fact that the frontend sends `type: \"adhoc\"` for both notebookand native SQL queries by inspecting the inner dataset-query type."[item]- (let [t (normalize-context-type (:type item))]- (if (and (= "adhoc" t) (native-query-item? item))- "native"- t)))+ (normalize-context-type (:type item)));;; Entity Formatting@@ -166,10 +162,31 @@(str id)(str/join ", " (map-indexed (fn [idx _] (str id "-" idx)) chart_configs)))))+(defn format-native-query+ "Format viewing `item`"+ [item]+ (assert (lib/native-only-query? (:query item))+ "Native MBQL5 query expected.")+ (te/lines+ "The user is currently in the SQL editor."+ (when (:id item)+ (te/field "Query ID" (:id item)))+ (te/field "Current SQL query"+ (te/code (lib/raw-native-query (:query item)) "sql"))+ (te/field "Database ID" (get-in item [:query :database]))+ (te/field "Database SQL engine" (:sql_engine item))+ (when-some [error (:error item)]+ (te/field "Query error" (te/code error)))+ (when-let [config-ids (format-chart-config-ids item)]+ (te/field "Chart Config IDs (for analyze_chart tool)" config-ids))+ (te/field "Tables used" (some->> (:used_tables item)+ (map format-entity)+ te/lines))))+(defmethod format-entity "question"[entity](if (native-query-item? entity)- (format-entity "native" entity)+ (format-native-query entity)(fetch-and-format entity"The user is currently looking at the results of a report:"#(entity-details/get-report-details {:report-id (:id entity)@@ -196,47 +213,16 @@;; Format adhoc query (notebook editor) viewing context.(defmethod format-entity "adhoc"[item]- (te/lines "The user is currently in the notebook editor viewing a query."- (te/field "Query ID" (:id item))- (te/field "Database ID" (get-in item [:query :database]))- (when-let [config-ids (format-chart-config-ids item)]- (te/field "Chart Config IDs (for analyze_chart tool)" config-ids))- (te/field "Tables used" (some->> (:used_tables item)- (map format-entity)- te/lines))))--;; Format native SQL query viewing context.-;; The :query field can be either a plain SQL string (legacy / explicit `type: "native"`)-;; or a dataset-query map (from the frontend with `type: "adhoc"`) where the actual SQL-;; lives at [:native :query].-(defmethod format-entity "native"- [item]- (let [query-val (:query item)- sql-text (cond- ;; Plain SQL string (legacy)- (string? query-val) query-val- ;; MLv2/pMBQL or legacy dataset-query map: normalize and use lib- (and (map? query-val) (:database query-val))- (try- (lib/raw-native-query (lib-be/normalize-query query-val))- (catch Exception _- ;; Fall back to manual extraction- (or (some :native (:stages query-val))- (get-in query-val [:native :query]))))- ;; Other map shapes- (map? query-val) (get-in query-val [:native :query])- :else nil)]- (te/lines- "The user is currently in the SQL editor."- (te/field "Query ID" (:id item))- (te/field "Current SQL query" (te/code sql-text "sql"))- (te/field "Database SQL engine" (:sql_engine item))- (te/field "Query error" (te/code (:error item)))- (when-let [config-ids (format-chart-config-ids item)]- (te/field "Chart Config IDs (for analyze_chart tool)" config-ids))- (te/field "Tables used" (some->> (:used_tables item)- (map format-entity)- te/lines)))))+ (if (native-query-item? item)+ (format-native-query item)+ (te/lines "The user is currently in the notebook editor viewing a query."+ (te/field "Query ID" (:id item))+ (te/field "Database ID" (get-in item [:query :database]))+ (when-let [config-ids (format-chart-config-ids item)]+ (te/field "Chart Config IDs (for analyze_chart tool)" config-ids))+ (te/field "Tables used" (some->> (:used_tables item)+ (map format-entity)+ te/lines)))))(defn- transform-query-source-text[source]diff --git a/src/metabase/metabot/context.clj b/src/metabase/metabot/context.cljindex 884f6ea473e..5e969b7ccdd 100644--- a/src/metabase/metabot/context.clj+++ b/src/metabase/metabot/context.clj@@ -7,6 +7,7 @@[metabase.config.core :as config][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.metabot.table-utils :as table-utils][metabase.transforms-base.util :as transforms-base.u][metabase.util.json :as json]@@ -45,8 +46,34 @@(.newLine w)(.newLine w)))))+(def item-types+ "Allowed values for the `:type` key of `:user_is_viewing` item."+ #{"adhoc"++ "question"+ "model"+ "metric"++ "document"+ "dashboard"+ "transform"+ "code_editor"})++(def ^:private item-type-schema+ "Schema for the `:type` key of `:user_is_viewing` item."+ (into [:enum] item-types))+(mr/def ::context- [:map-of :keyword :any])+ [:and+ [:map-of :keyword :any]+ [:map+ [:user_is_viewing+ {:optional true}+ [:vector [:map+ [:type item-type-schema]+ [:query+ {:optional true}+ ::lib.schema/query]]]]]])(defn- query-for-sql-parsing"Given an item in context, return the query if it is a native query or SQL transform that can have table usage parseddiff --git a/test/metabase/metabot/agent/user_context_test.clj b/test/metabase/metabot/agent/user_context_test.cljindex 396f7d02436..11aee7c7e26 100644--- a/test/metabase/metabot/agent/user_context_test.clj+++ b/test/metabase/metabot/agent/user_context_test.clj@@ -1,6 +1,9 @@(ns metabase.metabot.agent.user-context-test(:require[clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta][metabase.metabot.agent.user-context :as user-context][metabase.metabot.tools.entity-details :as entity-details][metabase.metabot.tools.shared.llm-representations :as llm-rep]@@ -41,25 +44,16 @@(is (= "postgresql" result))))(testing "extracts sql_engine from adhoc item with native dataset-query (frontend payload)"- (let [context {:user_is_viewing [{:type "adhoc"- :query {:type "native"- :native {:query "SELECT 1"}- :database 1}+ (let [context {:user_is_viewing [{:type "adhoc"+ :query (lib/native-query (mt/metadata-provider) "select 1"):sql_engine "PostgreSQL"}]}result (user-context/extract-sql-dialect context)](is (= "postgresql" result))))(testing "returns nil for adhoc notebook (MBQL) query"(let [context {:user_is_viewing [{:type "adhoc"- :query {:type "query"- :query {:source-table 1}- :database 1}}]}- result (user-context/extract-sql-dialect context)]- (is (nil? result))))-- (testing "extracts sql_engine from transform context"- (let [context {:user_is_viewing [{:type "transform"- :sql_engine "MySQL"}]}+ :query (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :venues))))}]}result (user-context/extract-sql-dialect context)](is (nil? result))))@@ -74,109 +68,89 @@(is (nil? result)))))(deftest format-viewing-context-test- (testing "formats adhoc notebook (MBQL) query context"- (is (=? #"(?s).*notebook editor.*Database ID: 1.*"- (user-context/format-viewing-context {:user_is_viewing [{:type "adhoc"- :query {:type "query"- :query {:source-table 1}- :database 1- :data_source "card__123"}}]}))))-- (testing "formats adhoc native SQL query from frontend (type: adhoc, query.type: native)"- (let [result (user-context/format-viewing-context- {:user_is_viewing [{:type "adhoc"- :query {:type "native"- :native {:query "SELECT * FROM orders WHERE total > 100"}- :database 1}- :sql_engine "PostgreSQL"- :error nil}]})]- (is (re-find #"SQL editor" result))- (is (re-find #"SELECT \* FROM orders WHERE total > 100" result))- (is (re-find #"PostgreSQL" result))))-- (testing "formats adhoc native SQL query with error"- (let [result (user-context/format-viewing-context- {:user_is_viewing [{:type "adhoc"- :query {:type "native"- :native {:query "SELECT * FROM invalid"}- :database 1}- :sql_engine "PostgreSQL"- :error "Table 'invalid' not found"}]})]- (is (re-find #"SQL editor" result))- (is (re-find #"SELECT \* FROM invalid" result))- (is (re-find #"Table 'invalid' not found" result))))-- (testing "formats explicit type: native context with SQL string (legacy)"- (let [result (user-context/format-viewing-context {:user_is_viewing [{:type "native"- :query "SELECT * FROM users"- :sql_engine "PostgreSQL"}]})]- (is (re-find #"SQL editor" result))- (is (re-find #"SELECT \* FROM users" result))- (is (re-find #"PostgreSQL" result))))-- (testing "formats native query with error (legacy)"- (let [context {:user_is_viewing [{:type "native"- :query "SELECT * FROM invalid"- :error "Table 'invalid' not found"}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"error" result))- (is (re-find #"Table 'invalid' not found" result))))-- (testing "formats transform context"- (let [context {:user_is_viewing [{:type "transform"- :id 123- :name "Daily Revenue"- :source_type "sql"}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"Transform" result))- (is (re-find #"Daily Revenue" result))- (is (re-find #"sql" result))))-- (testing "formats transform context with error"- (let [context {:user_is_viewing [{:type "transform"- :id 123- :name "Broken Revenue"- :source_type "native"- :error "ERROR: relation \"missing_table\" does not exist"}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"Transform error" result))- (is (re-find #"ERROR: relation \"missing_table\" does not exist" result))))-- (testing "formats code editor context"- (let [context {:user_is_viewing [{:type "code_editor"- :buffers [{:id "buffer1"- :source {:language "python"- :database_id 42}- :cursor {:line 10 :column 5}}]}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"code editor" result))- (is (re-find #"python" result))- (is (re-find #"Line 10" result))))-- (testing "formats code editor with selection"- (let [context {:user_is_viewing [{:type "code_editor"- :buffers [{:id "buffer1"- :source {:language "sql"- :database_id 42}- :cursor {:line 5 :column 0}- :selection {:start {:line 5 :column 0}- :end {:line 10 :column 20}- :text "SELECT * FROM foo"}}]}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"Selected lines:" result))- (is (re-find #"SELECT \* FROM foo" result))))-- (testing "formats code editor with no buffers"- (let [context {:user_is_viewing [{:type "code_editor"- :buffers []}]}- result (user-context/format-viewing-context context)]- (is (some? result))- (is (re-find #"no active buffers" result)))))+ (let [mp meta/metadata-provider]+ (testing "formats adhoc notebook (MBQL) query context"+ (is (=? (re-pattern+ (format "(?s).*notebook editor.*Database ID: %d.*"+ (meta/id)))+ (user-context/format-viewing-context+ {:user_is_viewing [{:type "adhoc"+ :query (lib/query mp (lib.metadata/table mp (meta/id :venues)))}]}))))++ (testing "formats adhoc native SQL query from frontend (type: adhoc, query.type: native)"+ (let [result (user-context/format-viewing-context+ {:user_is_viewing [{:type "adhoc"+ :query (lib/native-query mp "SELECT * FROM orders WHERE total > 100")+ :sql_engine "postgres"+ :error nil}]})]+ (is (re-find #"SQL editor" result))+ (is (re-find #"SELECT \* FROM orders WHERE total > 100" result))+ (is (re-find #"postgres" result))))++ (testing "formats adhoc native SQL query with error"+ (let [result (user-context/format-viewing-context+ {:user_is_viewing [{:type "adhoc"+ :query (lib/native-query mp "SELECT * FROM invalid")+ :sql_engine "postgres"+ :error "Table 'invalid' not found"}]})]+ (is (re-find #"SQL editor" result))+ (is (re-find #"SELECT \* FROM invalid" result))+ (is (re-find #"Table 'invalid' not found" result))))++ (testing "formats transform context"+ (let [context {:user_is_viewing [{:type "transform"+ :id 123+ :name "Daily Revenue"+ :source_type "sql"}]}+ result (user-context/format-viewing-context context)]+ (is (some? result))+ (is (re-find #"Transform" result))+ (is (re-find #"Daily Revenue" result))+ (is (re-find #"sql" result))))++ (testing "formats transform context with error"+ (let [context {:user_is_viewing [{:type "transform"+ :id 123+ :name "Broken Revenue"+ :source_type "native"+ :error "ERROR: relation \"missing_table\" does not exist"}]}+ result (user-context/format-viewing-context context)]+ (is (some? result))+ (is (re-find #"Transform error" result))+ (is (re-find #"ERROR: relation \"missing_table\" does not exist" result))))++ (testing "formats code editor context"+ (let [context {:user_is_viewing [{:type "code_editor"+ :buffers [{:id "buffer1"+ :source {:language "python"+ :database_id 42}+ :cursor {:line 10 :column 5}}]}]}+ result (user-context/format-viewing-context context)]+ (is (some? result))+ (is (re-find #"code editor" result))+ (is (re-find #"python" result))+ (is (re-find #"Line 10" result))))++ (testing "formats code editor with selection"+ (let [context {:user_is_viewing [{:type "code_editor"+ :buffers [{:id "buffer1"+ :source {:language "sql"+ :database_id 42}+ :cursor {:line 5 :column 0}+ :selection {:start {:line 5 :column 0}+ :end {:line 10 :column 20}+ :text "SELECT * FROM foo"}}]}]}+ result (user-context/format-viewing-context context)]+ (is (some? result))+ (is (re-find #"Selected lines:" result))+ (is (re-find #"SELECT \* FROM foo" result))))++ (testing "formats code editor with no buffers"+ (let [context {:user_is_viewing [{:type "code_editor"+ :buffers []}]}+ result (user-context/format-viewing-context context)]+ (is (some? result))+ (is (re-find #"no active buffers" result))))))(deftest format-viewing-context-test-2(testing "formats table entity"@@ -299,9 +273,9 @@(with-redefs [user-context/format-current-user-info (constantly "<user>Jane Doe</user>")](let [context {:current_time_with_timezone "2024-01-15T14:30:00-05:00":first_day_of_week "Monday"- :user_is_viewing [{:type "native"+ :user_is_viewing [{:type "adhoc":sql_engine "PostgreSQL"- :query "SELECT * FROM users"}]+ :query (lib/native-query (mt/metadata-provider) "SELECT * FROM users")}]:user_recently_viewed [{:type "table":id 123:name "users"}]}@@ -324,9 +298,7 @@:first_day_of_week "Monday":user_is_viewing [{:type "adhoc":sql_engine "PostgreSQL"- :query {:type "native"- :native {:query "SELECT * FROM users"}- :database 1}}]+ :query (lib/native-query (mt/metadata-provider) "SELECT * FROM users")}]:user_recently_viewed [{:type "table":id 123:name "users"}]}diff --git a/test/metabase/metabot/context_test.clj b/test/metabase/metabot/context_test.cljindex f344df068f5..6771d29636d 100644--- a/test/metabase/metabot/context_test.clj+++ b/test/metabase/metabot/context_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase.activity-feed.models.recent-views :as recent-views][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta][metabase.metabot.agent.user-context :as user-context][metabase.metabot.context :as context]@@ -257,12 +258,9 @@(deftest enhance-context-mbql-viewing-context-rendering-test(testing "MBQL adhoc query viewing context includes table name for LLM"(mt/with-test-user :rasta- (let [table-id (mt/id :orders)+ (let [mp (mt/metadata-provider)raw {:user_is_viewing [{:type "adhoc"- :query {:database (mt/id)- :lib/type "mbql/query"- :stages [{:lib/type "mbql.stage/mbql"- :source-table table-id}]}}]+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}]:current_time_with_timezone "2025-01-15T12:00:00+02:00"}enriched (context/create-context raw)uc-vars (user-context/enrich-context-for-template enriched)