The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[Metabot] Add v1 to v2 migration fns (#75662)
Sloan Sparger774531880af2
diff --git a/src/metabase/metabot/schema/migrate_v1_to_v2.clj b/src/metabase/metabot/schema/migrate_v1_to_v2.cljnew file mode 100644index 00000000000..85c76895706--- /dev/null+++ b/src/metabase/metabot/schema/migrate_v1_to_v2.clj@@ -0,0 +1,175 @@+(ns metabase.metabot.schema.migrate-v1-to-v2+ "Migration fns to migrate messages from the metabase.metabot.schema.v1 to the+ metabase.metabot.schema.v2 at-rest formats."+ (:require+ [malli.error :as me]+ [metabase.metabot.schema.v1 :as schema.v1]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(defn- assistant-placeholder?+ "An empty `metabot_message.data` value: assistant placeholder rows and aborted turns."+ [data]+ (and (sequential? data) (empty? data)))++(defn error->text+ "Extract a v1 tool-output `:error` text."+ [error]+ (cond+ (string? error) error+ (and (map? error)+ (string? (:message error))) (:message error)+ (map? error) (pr-str error)+ :else (str error)))++(defn- tool-call-entry->parts [{:keys [tool_calls]} outputs]+ (mapv (fn [{:keys [id name arguments]}]+ (let [output (get outputs id)+ input (try (json/decode+kw arguments)+ (catch Exception _ arguments))]+ (cond-> {:type (str "tool-" name)+ :toolCallId id+ :state (if output "output-available" "input-available")+ :input input}+ output (assoc :output {:output (:content output)}))))+ tool_calls))++(defn migrate-v1-external-ai-service->v2+ "Convert a v1 ai-service data array (`:_type`-keyed entries) to v2 parts.++ - `TEXT` becomes a text part+ - each call batched in a `TOOL_CALL` becomes its own `tool-<name>` part, merged with its+ matching `TOOL_RESULT` (orphan results are dropped)+ - `DATA` becomes a `data-<type>` part+ - `FINISH_MESSAGE` and `ERROR` are lifecycle signals with no history value and are dropped++ Throws on unrecognized entry types."+ [data]+ (let [outputs (into {}+ (comp (filter #(= "TOOL_RESULT" (:_type %)))+ (u/index-by :tool_call_id))+ data)]+ (->> data+ (remove #(= "TOOL_RESULT" (:_type %)))+ (mapcat (fn [entry]+ (case (:_type entry)+ "TEXT" [{:type "text" :text (:content entry)}]+ "TOOL_CALL" (tool-call-entry->parts entry outputs)+ "DATA" [{:type (str "data-" (:type entry))+ :data (:value entry)}]+ "FINISH_MESSAGE" []+ "ERROR" []+ (throw (ex-info "Unrecognized v1 ai-service entry type" {:entry entry})))))+ vec)))++(defn migrate-v1-native->v2+ "Convert a v1 native data array (`:type`-keyed message parts) to v2 parts.++ - each `tool-input` becomes a `tool-<function>` part, merged with its matching+ `tool-output` into an `output-available` or `output-error` state (orphan+ `tool-output`s with no matching `tool-input` are dropped)+ - `text` passes through without its `:id`+ - `data` becomes a `data-<data-type>` part+ - `error` entries signal turn failure, not conversation content, and are dropped++ Throws on unrecognized entry types."+ [data]+ (let [outputs (into {}+ (comp (filter #(= "tool-output" (:type %)))+ (u/index-by :id))+ data)]+ (->> data+ (mapcat (fn [entry]+ (case (:type entry)+ "text" [{:type "text" :text (:text entry)}]+ "tool-input" (let [output (get outputs (:id entry))+ error (:error output)]+ [(cond-> {:type (str "tool-" (:function entry))+ :toolCallId (:id entry)+ :state (cond+ (some? error) "output-error"+ (some? output) "output-available"+ :else "input-available")+ :input (:arguments entry)}+ ;; errored tool-outputs do co-occur with `:result`, but only because+ ;; `persistence/strip-tool-output-bloat` rewrites the absent result of an+ ;; errored call to `{}` — there is never a real result to preserve+ (and (some? output) (nil? error))+ ;; the result map passes through verbatim: its keys (`:output`,+ ;; `:structured-output`/`:structured_output`) keep their v1 spelling+ ;; inside the opaque v2 `:output`, matching what consumers like+ ;; `metabot-analytics.queries` already read+ (assoc :output (:result output))++ (some? error)+ (assoc :errorText (error->text error)))])+ "tool-output" []+ "data" [{:type (str "data-" (:data-type entry))+ :data (:data entry)}]+ "error" []+ (throw (ex-info "Unrecognized v1 native entry type" {:entry entry})))))+ vec)))++(defn migrate-v1-user-message->v2+ "Convert a v1 user-message data array to v2 parts."+ [data]+ (mapv (fn [{:keys [content]}] {:type "text" :text content}) data))++(defn migrate-v1->v2+ "Convert a v1 `metabot_message.data` value to the v2 format.+ Empty arrays (assistant placeholders and aborted turns) pass through.+ Throws on values that do not satisfy any v1 schema."+ [data]+ (cond+ (assistant-placeholder? data) data+ (mr/validate ::schema.v1/ai-service-data data) (migrate-v1-external-ai-service->v2 data)+ (mr/validate ::schema.v1/native-data data) (migrate-v1-native->v2 data)+ (mr/validate ::schema.v1/user-message-data data) (migrate-v1-user-message->v2 data)+ :else+ ;; explain only against the format the row resembles (by its entries' dispatch tag),+ ;; rather than dumping a humanized mismatch for all three v1 schemas+ (let [entry (when (sequential? data) (first data))+ schema (cond+ (:_type entry) ::schema.v1/ai-service-data+ (:type entry) ::schema.v1/native-data+ (= "user" (:role entry)) ::schema.v1/user-message-data)]+ (throw (ex-info "Unrecognized v1 storage format"+ (cond-> {:data data}+ schema (assoc :explanation (me/humanize (mr/explain schema data)))))))))++(comment+ ;; validate the v1->v2 conversion against a CSV dump of metabot_message table+ (require '[clojure.data.csv :as csv]+ '[clojure.java.io :as io]+ '[metabase.metabot.schema.v2])++ (defn validate-conversion-csv [path]+ (with-open [reader (io/reader path)]+ (let [[header & rows] (csv/read-csv reader)+ column-index (fn [column] (first (keep-indexed #(when (= column %2) %1) header)))+ id-idx (column-index "id")+ data-idx (column-index "data")+ explain (mr/explainer :metabase.metabot.schema.v2/message-data)]+ (reduce (fn [acc row]+ (let [raw (nth row data-idx)+ v1 (try (json/decode+kw raw) (catch Exception e e))+ migrated (if (instance? Exception v1)+ v1+ (try (migrate-v1->v2 v1) (catch Exception e e)))+ error (cond+ (instance? Exception v1) {:json-parse (ex-message v1)}+ (instance? Exception migrated) {:migrate (ex-message migrated)+ :data (ex-data migrated)}+ :else (some-> (explain migrated) me/humanize))]+ (cond-> (update acc :row-count inc)+ error (-> (update :failure-count inc)+ (update :failures conj {:message-id (when id-idx (nth row id-idx))+ :message (if (instance? Exception v1) raw v1)+ :error error})))))+ {:row-count 0 :failure-count 0 :failures []}+ rows))))++ (validate-conversion-csv "./metabot_message_dump.csv"))diff --git a/src/metabase/metabot/schema/v2.clj b/src/metabase/metabot/schema/v2.cljindex d0bf74cd2ed..43efb282fba 100644--- a/src/metabase/metabot/schema/v2.clj+++ b/src/metabase/metabot/schema/v2.clj@@ -402,3 +402,8 @@[:role [:enum "system" "user" "assistant"]][:metadata {:optional true} :any][:parts [:sequential {:min 1} ::ui-message-part]]])++(mr/def ::message-data+ "A whole `metabot_message.data` value in the v2 format: the at-rest projection of a+ message's `UIMessagePart`s. Assistant placeholder rows are `[]`."+ [:sequential ::ui-message-part])diff --git a/test/metabase/metabot/schema/migrate_v1_to_v2_test.clj b/test/metabase/metabot/schema/migrate_v1_to_v2_test.cljnew file mode 100644index 00000000000..111288a1be0--- /dev/null+++ b/test/metabase/metabot/schema/migrate_v1_to_v2_test.clj@@ -0,0 +1,152 @@+(ns metabase.metabot.schema.migrate-v1-to-v2-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.metabot.schema.migrate-v1-to-v2 :as migrate]+ [metabase.metabot.schema.v2 :as schema.v2]+ [metabase.util.malli.registry :as mr]))++(deftest ^:parallel migrate-v1-external-ai-service->v2-test+ (testing "TEXT becomes a text part"+ (is (= [{:type "text" :text "hello"}]+ (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TEXT" :content "hello"}]))))+ (testing "batched TOOL_CALL entries split into one part per call, merged with TOOL_RESULTs in position"+ (is (= [{:type "text" :text "before"}+ {:type "tool-search" :toolCallId "a" :state "output-available" :input {:q "x"} :output {:output "r1"}}+ {:type "tool-browse" :toolCallId "b" :state "output-available" :input {} :output {:output "r2"}}+ {:type "text" :text "after"}]+ (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TEXT" :content "before"}+ {:role "assistant" :_type "TOOL_CALL"+ :tool_calls [{:id "a" :name "search" :arguments "{\"q\":\"x\"}"}+ {:id "b" :name "browse" :arguments "{}"}]}+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "a" :content "r1"}+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "b" :content "r2"}+ {:role "assistant" :_type "TEXT" :content "after"}]))))+ (testing "tool name underscores are preserved in :type"+ (is (= "tool-create_sql_query"+ (-> (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TOOL_CALL"+ :tool_calls [{:id "tc1" :name "create_sql_query" :arguments "{}"}]}])+ first :type))))+ (testing "TOOL_CALL without a matching result stays input-available with no :output"+ (let [[part] (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TOOL_CALL"+ :tool_calls [{:id "tc1" :name "search" :arguments "{}"}]}])]+ (is (= "input-available" (:state part)))+ (is (not (contains? part :output)))))+ (testing "malformed :arguments JSON falls back to the raw string"+ (is (= "not-json"+ (-> (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TOOL_CALL"+ :tool_calls [{:id "tc1" :name "search" :arguments "not-json"}]}])+ first :input))))+ (testing "orphan TOOL_RESULT is dropped"+ (is (= [] (migrate/migrate-v1-external-ai-service->v2+ [{:role "tool" :_type "TOOL_RESULT" :tool_call_id "nope" :content "x"}]))))+ (testing "DATA becomes a data-<type> part, dropping :version"+ (is (= [{:type "data-navigate_to" :data "/metric/17991"}]+ (migrate/migrate-v1-external-ai-service->v2+ [{:_type "DATA" :type "navigate_to" :version 1 :value "/metric/17991"}]))))+ (testing "FINISH_MESSAGE and ERROR are dropped"+ (is (= [{:type "text" :text "hi"}]+ (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "TEXT" :content "hi"}+ {:role "assistant" :_type "ERROR" :content "upstream provider error"}+ {:role "assistant" :_type "FINISH_MESSAGE" :finish_reason "stop" :usage {}}]))))+ (testing "unknown :_type throws"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 ai-service entry type"+ (migrate/migrate-v1-external-ai-service->v2+ [{:role "assistant" :_type "REASONING" :content "hmm"}])))))++(deftest ^:parallel migrate-v1-native->v2-test+ (let [tool-input {:type "tool-input" :id "tc1" :function "search" :arguments {:q "x"}}]+ (testing "tool-input merged with tool-output becomes output-available carrying the result map"+ (is (= [{:type "tool-search" :toolCallId "tc1" :state "output-available" :input {:q "x"} :output {:output "rows"}}]+ (migrate/migrate-v1-native->v2+ [tool-input+ {:type "tool-output" :id "tc1" :result {:output "rows"} :error nil :duration-ms 3}]))))+ (testing "empty result map still produces the :output key"+ (let [[part] (migrate/migrate-v1-native->v2+ [tool-input {:type "tool-output" :id "tc1" :result {}}])]+ (is (= "output-available" (:state part)))+ (is (contains? part :output))+ (is (= {} (:output part)))))+ (testing "tool-output with a string :error becomes output-error with :errorText and no :output"+ (let [[part] (migrate/migrate-v1-native->v2+ [tool-input {:type "tool-output" :id "tc1" :result {} :error "boom"}])]+ (is (= "output-error" (:state part)))+ (is (= "boom" (:errorText part)))+ (is (not (contains? part :output)))))+ (testing "map-shaped :error extracts :message"+ (is (= "bad"+ (-> (migrate/migrate-v1-native->v2+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:message "bad"}}])+ first :errorText))))+ (testing "map-shaped :error without :message falls back to pr-str"+ (is (= "{:code 500}"+ (-> (migrate/migrate-v1-native->v2+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:code 500}}])+ first :errorText))))+ (testing "map-shaped :error with a non-string :message falls back to pr-str"+ (is (= "{:message {:code 500}}"+ (-> (migrate/migrate-v1-native->v2+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:message {:code 500}}}])+ first :errorText))))+ (testing "tool-input without a matching output stays input-available"+ (let [[part] (migrate/migrate-v1-native->v2 [tool-input])]+ (is (= "input-available" (:state part)))+ (is (not (contains? part :output)))+ (is (not (contains? part :errorText))))))+ (testing "text passes through with :id stripped"+ (is (= [{:type "text" :text "Found it"}]+ (migrate/migrate-v1-native->v2+ [{:type "text" :id "t1" :text "Found it"}]))))+ (testing "data entries convert like external DATA entries"+ (is (= [{:type "data-navigate_to" :data "/question/1"}]+ (migrate/migrate-v1-native->v2+ [{:type "data" :data-type "navigate_to" :version 1 :data "/question/1"}]))))+ (testing "error entries are dropped"+ (is (= [] (migrate/migrate-v1-native->v2+ [{:type "error" :error {:message "boom"}}]))))+ (testing "unknown :type throws"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 native entry type"+ (migrate/migrate-v1-native->v2+ [{:type "reasoning" :text "hmm"}])))))++(deftest ^:parallel migrate-v1-user-message->v2-test+ (testing "user messages convert to text parts"+ (is (= [{:type "text" :text "Do we have data on orders"}]+ (migrate/migrate-v1-user-message->v2+ [{:role "user" :content "Do we have data on orders"}])))))++(deftest ^:parallel migrate-v1->v2-invalid-data-test+ (testing "a row resembling a v1 shape but failing its schema throws, explaining against the format it resembles"+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 storage format"+ (migrate/migrate-v1->v2 [{:type "text"}])))]+ (is (= [{:text ["missing required key"]}]+ (:explanation (ex-data e)))))))++(deftest ^:parallel migrated-rows-validate-test+ (testing "migrating every v1 row shape should produce valid v2 message data"+ ;; one row per v1 storage format — ai-service, native, user message, and empty —+ ;; collectively exercising every entry type a format can hold+ (let [v1-rows [[{:role "assistant" :_type "TEXT" :content "hello"}+ {:role "assistant" :_type "TOOL_CALL"+ :tool_calls [{:id "a" :name "create_sql_query" :arguments "{\"q\":\"x\"}"}+ {:id "b" :name "browse" :arguments "{}"}]}+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "a" :content "r1"}+ {:_type "DATA" :type "navigate_to" :version 1 :value "/q/1"}+ {:role "assistant" :_type "FINISH_MESSAGE" :finish_reason "stop" :usage {}}]+ [{:type "tool-input" :id "tc1" :function "search" :arguments {:q "x"}}+ {:type "tool-output" :id "tc1" :result {:output "rows"} :error nil :duration-ms 3}+ {:type "tool-input" :id "tc2" :function "browse" :arguments {}}+ {:type "tool-output" :id "tc2" :result {} :error {:message "boom"}}+ {:type "tool-input" :id "tc3" :function "search" :arguments {:q "y"}}+ {:type "text" :id "t1" :text "Found it"}+ {:type "data" :data-type "state" :version 1 :data {}}+ {:type "error" :error {:message "boom"}}]+ [{:role "user" :content "Do we have data on orders"}]+ []]]+ (is (nil? (mr/explain [:sequential ::schema.v2/message-data]+ (mapv migrate/migrate-v1->v2 v1-rows)))))))