The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Inject context in user message (#71045)
lbrdnkb1a2673e1a9f
diff --git a/resources/metabot/prompts/message_injection.selmer b/resources/metabot/prompts/message_injection.selmernew file mode 100644index 00000000000..5fe37395118--- /dev/null+++ b/resources/metabot/prompts/message_injection.selmer@@ -0,0 +1,17 @@+<context>++Current date: {{ current_time }}+In the system, the first day of the week (index 1) is: {{ first_day_of_week }}+{% if current_user_info %}++<user>+Here is some information about the user:+{{ current_user_info|safe }}+</user>+{% endif %}+{% if viewing_context %}++{{ viewing_context|safe }}+{% endif %}++</context>diff --git a/resources/metabot/prompts/system/internal.selmer b/resources/metabot/prompts/system/internal.selmerindex f500dd0153d..f9095877662 100644--- a/resources/metabot/prompts/system/internal.selmer+++ b/resources/metabot/prompts/system/internal.selmer@@ -542,23 +542,6 @@ Here's your [Sales Metric Cube](metabase://query/456).# Important Context-Current date: {{ current_time }}-In the system, the first day of the week (index 1) is: {{ first_day_of_week }}-{% if current_user_info %}--<user_context>-Here is some information about the user:-{{ current_user_info|safe }}-{% if viewing_context %}-{{ viewing_context|safe }}-{% endif %}-</user_context>-{% elif viewing_context %}--<user_context>-{{ viewing_context|safe }}-</user_context>-{% endif %}{% if recent_views %}{{ recent_views|safe }}diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex cc065c57e7d..9a47839afb7 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -227,7 +227,7 @@[memory context profile tools iteration tracking-opts link-registry-atom](let [model (:model profile)system-msg (messages/build-system-message context profile tools)- input-parts (-> (messages/build-message-history memory)+ input-parts (-> (messages/build-message-history context memory)(invert-links @link-registry-atom))llm-opts (cond-> {}(:required-tool-call? profile) (assoc :tool-choice "required"))]diff --git a/src/metabase/metabot/agent/messages.clj b/src/metabase/metabot/agent/messages.cljindex 04587fc0493..4d3a8003f3a 100644--- a/src/metabase/metabot/agent/messages.clj+++ b/src/metabase/metabot/agent/messages.clj@@ -135,9 +135,23 @@(->> (:parts step)(filter #(#{:text :tool-input :tool-output} (:type %)))))+(defn- messages-with-injected-context+ "Returns messages from memory and injects context into the most recent one."+ [context memory]+ (let [all-messages (memory/get-input-messages memory)+ [input-messages [most-recent-message]] (split-at (dec (count all-messages)) all-messages)]+ (if-not (and (= :user (-> most-recent-message :role))+ (< 0 (count all-messages)))+ all-messages+ (let [enriched-context (user-context/enrich-context-for-template context)]+ (conj (vec input-messages)+ (update most-recent-message :content (partial prompts/inject-context enriched-context)))))))+(defn build-message-history"Build the conversation history as a flat sequence of AISDK parts.+ Injects the context into the most recent user message.+Returns a vector of items that are either:- `{:role :user, :content \"...\"}` for user messages- `{:type :text, :text \"...\"}` for assistant text@@ -145,8 +159,8 @@- `{:type :tool-output, :id ..., :result ...}` for tool resultsEach LLM adapter converts this to its own wire format."- [memory]- (let [input-messages (memory/get-input-messages memory)+ [context memory]+ (let [input-messages (messages-with-injected-context context memory)steps (memory/get-steps memory)input-parts (mapcat input-message->parts input-messages)step-parts (mapcat step->parts steps)diff --git a/src/metabase/metabot/agent/prompts.clj b/src/metabase/metabot/agent/prompts.cljindex d0c132910f8..d79ff7d02a1 100644--- a/src/metabase/metabot/agent/prompts.clj+++ b/src/metabase/metabot/agent/prompts.clj@@ -155,6 +155,13 @@"llm_representations.selmer"load-llm-representations-template)))+(defn get-cached-message-injection-template+ "Get the message injection template from cache or load it."+ []+ (:template (cached-load-template "message-injection"+ "message_injection.selmer"+ load-llm-representations-template)))+(defn clear-cache!"Clear the template cache. Useful for development/testing."[]@@ -228,6 +235,17 @@(log/error "System prompt template not found:" template-name)"You are Metabot, a data analysis assistant for Metabase."))))+(defn inject-context+ "Prepends a formatted context to a string, message."+ [context str*]+ (let [;; Injection is performed only when the hardcoded keys are present in in context to avoid+ ;; insertion e.g. insertion of blank xml tags.+ injection (when (some #(string? (not-empty (get context %)))+ [:viewing_context :current_time :current_user_info])+ (selmer/render (get-cached-message-injection-template)+ context))]+ (str injection str*)))+(comment;; Development examplesdiff --git a/test/metabase/metabot/agent/messages_test.clj b/test/metabase/metabot/agent/messages_test.cljindex 51779f13673..f88ad3b816d 100644--- a/test/metabase/metabot/agent/messages_test.clj+++ b/test/metabase/metabot/agent/messages_test.clj@@ -2,8 +2,10 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase.lib.core :as lib][metabase.metabot.agent.memory :as memory]- [metabase.metabot.agent.messages :as messages]))+ [metabase.metabot.agent.messages :as messages]+ [metabase.test :as mt]));;; ──────────────────────────────────────────────────────────────────;;; input-message->parts@@ -83,28 +85,32 @@(deftest ^:parallel build-message-history-test(testing "builds history from user messages only"- (is (=? [{:role :user :content "Hello"}]+ (is (=? [{:role :user :content #(str/ends-with? % "Hello")}](messages/build-message-history+ {}(memory/initialize [{:role :user :content "Hello"}] {})))))(testing "includes assistant text from input"(is (=? [{:role :user :content "Hello"}{:type :text :text "Hi there"}](messages/build-message-history+ {}(memory/initialize [{:role :user :content "Hello"}{:role :assistant :content "Hi there"}] {})))))(testing "includes step parts from memory"- (is (=? [{:role :user :content "Hello"}+ (is (=? [{:role :user :content #(str/ends-with? % "Hello")}{:type :text :text "Response text"}](messages/build-message-history+ {}(-> (memory/initialize [{:role :user :content "Hello"}] {})(memory/add-step [{:type :text :text "Response text"}]))))))(testing "includes tool calls from steps"- (is (=? [{:role :user :content "Search for revenue"}+ (is (=? [{:role :user :content #(str/ends-with? % "Search for revenue")}{:type :tool-input :id "t1" :function "search" :arguments {:query "revenue"}}](messages/build-message-history+ {}(-> (memory/initialize [{:role :user :content "Search for revenue"}] {})(memory/add-step [{:type :tool-input:id "t1"@@ -112,29 +118,32 @@:arguments {:query "revenue"}}]))))))(testing "includes tool results from steps"- (is (=? [{:role :user :content "Search"}+ (is (=? [{:role :user :content #(str/ends-with? % "Search")}{:type :tool-input :id "t1" :function "search"}{:type :tool-output :id "t1"}](messages/build-message-history+ {}(-> (memory/initialize [{:role :user :content "Search"}] {})(memory/add-step [{:type :tool-input :id "t1" :function "search" :arguments {:query "test"}}])(memory/add-step [{:type :tool-output :id "t1" :result {:data []}}]))))))(testing "handles multiple iterations"- (is (=? [{:role :user :content "Hello"}+ (is (=? [{:role :user :content #(str/ends-with? % "Hello")}{:type :tool-input :id "t1"}{:type :tool-output :id "t1"}{:type :text :text "Found results"}](messages/build-message-history+ {}(-> (memory/initialize [{:role :user :content "Hello"}] {})(memory/add-step [{:type :tool-input :id "t1" :function "search" :arguments {}}])(memory/add-step [{:type :tool-output :id "t1" :result {:data []}}])(memory/add-step [{:type :text :text "Found results"}]))))))(testing "filters out non-message parts from steps"- (is (=? [{:role :user :content "Hello"}+ (is (=? [{:role :user :content #(str/ends-with? % "Hello")}{:type :text :text "Response"}](messages/build-message-history+ {}(-> (memory/initialize [{:role :user :content "Hello"}] {})(memory/add-step [{:type :start :messageId "m1"}{:type :text :text "Response"}@@ -147,12 +156,116 @@{:type :tool-input :id "t1" :function "search"}{:type :tool-output :id "t1"}](messages/build-message-history+ {}(memory/initialize [{:role :user :content "Hello"}{:role :assistant :content "I'll search for that."}{:role :assistant :tool_calls [{:id "t1" :name "search" :arguments "{}"}]}{:role :tool :tool_call_id "t1" :content "results"}]{}))))))+;;; ──────────────────────────────────────────────────────────────────+;;; build-message-history — context injection+;;; ──────────────────────────────────────────────────────────────────++(defn- last-user-content+ "Return the :content of the last user message from build-message-history output."+ [parts]+ (->> parts (filter #(= :user (:role %))) last :content))++(deftest ^:parallel context-injection-basic-test+ (testing "injects <context> block into the last user message"+ (let [content (last-user-content+ (messages/build-message-history+ {}+ (memory/initialize [{:role :user :content "Hello"}] {})))]+ (is (str/starts-with? content "<context>"))+ (is (str/ends-with? content "Hello"))+ (is (str/includes? content "Current date:"))+ (is (str/includes? content "first day of the week")))))++(deftest ^:parallel context-injection-skips-non-user-last-message-test+ (testing "does not inject when last message is not :user"+ (let [parts (messages/build-message-history+ {}+ (memory/initialize [{:role :user :content "Hello"}+ {:role :assistant :content "Reply"}] {}))]+ (is (= "Hello" (:content (first parts)))))))++(deftest ^:parallel context-injection-targets-last-user-only-test+ (testing "injects into last user message only, not earlier ones"+ (let [parts (messages/build-message-history+ {}+ (memory/initialize [{:role :user :content "First"}+ {:role :assistant :content "Reply"}+ {:role :user :content "Second"}] {}))]+ (is (= "First" (:content (first parts))))+ (let [last-user (last-user-content parts)]+ (is (str/starts-with? last-user "<context>"))+ (is (str/ends-with? last-user "Second"))))))++(deftest ^:parallel context-injection-does-not-affect-assistant-parts-test+ (testing "context injection does not affect non-user parts"+ (let [parts (messages/build-message-history+ {:user_is_viewing [{:type "dashboard" :id 1 :name "Sales"}]}+ (-> (memory/initialize [{:role :user :content "Hello"}] {})+ (memory/add-step [{:type :text :text "Response"}])))]+ (is (= "Response" (:text (second parts)))))))++(deftest ^:parallel context-injection-first-day-of-week-test+ (testing "includes first_day_of_week from context"+ (let [content (last-user-content+ (messages/build-message-history+ {:first_day_of_week "Monday"}+ (memory/initialize [{:role :user :content "Hi"}] {})))]+ (is (str/includes? content "Monday"))))++ (testing "default first_day_of_week is Sunday"+ (let [content (last-user-content+ (messages/build-message-history+ {}+ (memory/initialize [{:role :user :content "Hi"}] {})))]+ (is (str/includes? content "Sunday")))))++(deftest ^:parallel context-injection-viewing-dashboard-test+ (testing "includes viewing context when user is viewing a dashboard"+ (let [content (last-user-content+ (messages/build-message-history+ {:user_is_viewing [{:type "dashboard" :id 1 :name "Sales"}]}+ (memory/initialize [{:role :user :content "Hi"}] {})))]+ (is (str/includes? content "Sales")))))++(deftest ^:parallel context-injection-viewing-native-query-test+ (testing "includes SQL and schema when user is viewing a native query"+ (let [mp (mt/metadata-provider)+ context {:user_is_viewing [{:type "adhoc",+ :query+ (lib/native-query mp "SELECT * FROM orders WHERE status = 'paid'")+ :sql_engine "postgres"+ :used_tables+ [{:description nil,+ :database_id 2,+ :name "orders",+ :fields+ [{:field_id "t23-0" :name "id" :display_name "ID" :type :number :database_type "int4" :semantic_type "pk"}+ {:field_id "t24-0" :name "status" :display_name "Status" :type :number :database_type "int4"}+ {:field_id "t25-0" :name "total" :display_name "Total" :type :number :database_type "int4"}]+ :type :table+ :database_schema "public",+ :display_name "Orders"}],+ :id "6ef8bcf9-383d-449f-827f-501c4d9b3564"}+ {:type "code_editor",+ :buffers [{:id "qb", :source {:language "sql", :database_id 2}, :cursor {:line 0, :column 0}}],+ :id "f4f07783-9276-403f-af5f-b9e7bd96fc88"}]}+ content (last-user-content+ (messages/build-message-history+ context+ (memory/initialize [{:role :user :content "Fix my query"}] {})))]+ (is (str/includes? content "SQL editor"))+ (is (str/includes? content "SELECT * FROM orders WHERE status = 'paid'"))+ (is (str/includes? content "public.orders"))+ (is (str/includes? content "id, status, total"))+ (is (str/ends-with? content "Fix my query")))))+(deftest ^:parallel build-system-message-test(testing "builds basic system message"(let [profile {:prompt-template "internal.selmer"diff --git a/test/metabase/metabot/agent/prompts_test.clj b/test/metabase/metabot/agent/prompts_test.cljindex 4fab5640f18..4ba35d9b06e 100644--- a/test/metabase/metabot/agent/prompts_test.clj+++ b/test/metabase/metabot/agent/prompts_test.clj@@ -172,12 +172,12 @@(is (not (str/includes? content "{%raw%}")))(is (not (str/includes? content "{% safe %}")))))- (testing "includes current user info when provided"+ (testing "current user info is not in system message (moved to message injection)"(let [profile {:prompt-template "internal.selmer"}context {:current_time "2024-01-15 14:30:00":current_user_info "<user><name>Jane Doe</name></user>"}tools {}content (prompts/build-system-message-content profile context tools)](is (some? content))- (is (str/includes? content "Here is some information about the user:"))- (is (str/includes? content "<user><name>Jane Doe</name></user>")))))+ (is (not (str/includes? content "Here is some information about the user:")))+ (is (not (str/includes? content "<user><name>Jane Doe</name></user>"))))))