Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[Metabot] Add v1 to v2 migration fns (#75662)

Sloan Sparger774531880af2
diff --git a/src/metabase/metabot/schema/migrate_v1_to_v2.clj b/src/metabase/metabot/schema/migrate_v1_to_v2.clj
new file mode 100644
index 00000000000..85c76895706
--- /dev/null
+++ b/src/metabase/metabot/schema/migrate_v1_to_v2.clj
@@ -0,0 +1,175 @@
+(ns metabase.metabot.schema.migrate-v1-to-v2
+ "Migration fns to migrate messages from the metabase.metabot.schema.v1 to the
+ metabase.metabot.schema.v2 at-rest formats."
+ (:require
+ [malli.error :as me]
+ [metabase.metabot.schema.v1 :as schema.v1]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(defn- assistant-placeholder?
+ "An empty `metabot_message.data` value: assistant placeholder rows and aborted turns."
+ [data]
+ (and (sequential? data) (empty? data)))
+
+(defn error->text
+ "Extract a v1 tool-output `:error` text."
+ [error]
+ (cond
+ (string? error) error
+ (and (map? error)
+ (string? (:message error))) (:message error)
+ (map? error) (pr-str error)
+ :else (str error)))
+
+(defn- tool-call-entry->parts [{:keys [tool_calls]} outputs]
+ (mapv (fn [{:keys [id name arguments]}]
+ (let [output (get outputs id)
+ input (try (json/decode+kw arguments)
+ (catch Exception _ arguments))]
+ (cond-> {:type (str "tool-" name)
+ :toolCallId id
+ :state (if output "output-available" "input-available")
+ :input input}
+ output (assoc :output {:output (:content output)}))))
+ tool_calls))
+
+(defn migrate-v1-external-ai-service->v2
+ "Convert a v1 ai-service data array (`:_type`-keyed entries) to v2 parts.
+
+ - `TEXT` becomes a text part
+ - each call batched in a `TOOL_CALL` becomes its own `tool-<name>` part, merged with its
+ matching `TOOL_RESULT` (orphan results are dropped)
+ - `DATA` becomes a `data-<type>` part
+ - `FINISH_MESSAGE` and `ERROR` are lifecycle signals with no history value and are dropped
+
+ Throws on unrecognized entry types."
+ [data]
+ (let [outputs (into {}
+ (comp (filter #(= "TOOL_RESULT" (:_type %)))
+ (u/index-by :tool_call_id))
+ data)]
+ (->> data
+ (remove #(= "TOOL_RESULT" (:_type %)))
+ (mapcat (fn [entry]
+ (case (:_type entry)
+ "TEXT" [{:type "text" :text (:content entry)}]
+ "TOOL_CALL" (tool-call-entry->parts entry outputs)
+ "DATA" [{:type (str "data-" (:type entry))
+ :data (:value entry)}]
+ "FINISH_MESSAGE" []
+ "ERROR" []
+ (throw (ex-info "Unrecognized v1 ai-service entry type" {:entry entry})))))
+ vec)))
+
+(defn migrate-v1-native->v2
+ "Convert a v1 native data array (`:type`-keyed message parts) to v2 parts.
+
+ - each `tool-input` becomes a `tool-<function>` part, merged with its matching
+ `tool-output` into an `output-available` or `output-error` state (orphan
+ `tool-output`s with no matching `tool-input` are dropped)
+ - `text` passes through without its `:id`
+ - `data` becomes a `data-<data-type>` part
+ - `error` entries signal turn failure, not conversation content, and are dropped
+
+ Throws on unrecognized entry types."
+ [data]
+ (let [outputs (into {}
+ (comp (filter #(= "tool-output" (:type %)))
+ (u/index-by :id))
+ data)]
+ (->> data
+ (mapcat (fn [entry]
+ (case (:type entry)
+ "text" [{:type "text" :text (:text entry)}]
+ "tool-input" (let [output (get outputs (:id entry))
+ error (:error output)]
+ [(cond-> {:type (str "tool-" (:function entry))
+ :toolCallId (:id entry)
+ :state (cond
+ (some? error) "output-error"
+ (some? output) "output-available"
+ :else "input-available")
+ :input (:arguments entry)}
+ ;; errored tool-outputs do co-occur with `:result`, but only because
+ ;; `persistence/strip-tool-output-bloat` rewrites the absent result of an
+ ;; errored call to `{}` — there is never a real result to preserve
+ (and (some? output) (nil? error))
+ ;; the result map passes through verbatim: its keys (`:output`,
+ ;; `:structured-output`/`:structured_output`) keep their v1 spelling
+ ;; inside the opaque v2 `:output`, matching what consumers like
+ ;; `metabot-analytics.queries` already read
+ (assoc :output (:result output))
+
+ (some? error)
+ (assoc :errorText (error->text error)))])
+ "tool-output" []
+ "data" [{:type (str "data-" (:data-type entry))
+ :data (:data entry)}]
+ "error" []
+ (throw (ex-info "Unrecognized v1 native entry type" {:entry entry})))))
+ vec)))
+
+(defn migrate-v1-user-message->v2
+ "Convert a v1 user-message data array to v2 parts."
+ [data]
+ (mapv (fn [{:keys [content]}] {:type "text" :text content}) data))
+
+(defn migrate-v1->v2
+ "Convert a v1 `metabot_message.data` value to the v2 format.
+ Empty arrays (assistant placeholders and aborted turns) pass through.
+ Throws on values that do not satisfy any v1 schema."
+ [data]
+ (cond
+ (assistant-placeholder? data) data
+ (mr/validate ::schema.v1/ai-service-data data) (migrate-v1-external-ai-service->v2 data)
+ (mr/validate ::schema.v1/native-data data) (migrate-v1-native->v2 data)
+ (mr/validate ::schema.v1/user-message-data data) (migrate-v1-user-message->v2 data)
+ :else
+ ;; explain only against the format the row resembles (by its entries' dispatch tag),
+ ;; rather than dumping a humanized mismatch for all three v1 schemas
+ (let [entry (when (sequential? data) (first data))
+ schema (cond
+ (:_type entry) ::schema.v1/ai-service-data
+ (:type entry) ::schema.v1/native-data
+ (= "user" (:role entry)) ::schema.v1/user-message-data)]
+ (throw (ex-info "Unrecognized v1 storage format"
+ (cond-> {:data data}
+ schema (assoc :explanation (me/humanize (mr/explain schema data)))))))))
+
+(comment
+ ;; validate the v1->v2 conversion against a CSV dump of metabot_message table
+ (require '[clojure.data.csv :as csv]
+ '[clojure.java.io :as io]
+ '[metabase.metabot.schema.v2])
+
+ (defn validate-conversion-csv [path]
+ (with-open [reader (io/reader path)]
+ (let [[header & rows] (csv/read-csv reader)
+ column-index (fn [column] (first (keep-indexed #(when (= column %2) %1) header)))
+ id-idx (column-index "id")
+ data-idx (column-index "data")
+ explain (mr/explainer :metabase.metabot.schema.v2/message-data)]
+ (reduce (fn [acc row]
+ (let [raw (nth row data-idx)
+ v1 (try (json/decode+kw raw) (catch Exception e e))
+ migrated (if (instance? Exception v1)
+ v1
+ (try (migrate-v1->v2 v1) (catch Exception e e)))
+ error (cond
+ (instance? Exception v1) {:json-parse (ex-message v1)}
+ (instance? Exception migrated) {:migrate (ex-message migrated)
+ :data (ex-data migrated)}
+ :else (some-> (explain migrated) me/humanize))]
+ (cond-> (update acc :row-count inc)
+ error (-> (update :failure-count inc)
+ (update :failures conj {:message-id (when id-idx (nth row id-idx))
+ :message (if (instance? Exception v1) raw v1)
+ :error error})))))
+ {:row-count 0 :failure-count 0 :failures []}
+ rows))))
+
+ (validate-conversion-csv "./metabot_message_dump.csv"))
diff --git a/src/metabase/metabot/schema/v2.clj b/src/metabase/metabot/schema/v2.clj
index d0bf74cd2ed..43efb282fba 100644
--- a/src/metabase/metabot/schema/v2.clj
+++ b/src/metabase/metabot/schema/v2.clj
@@ -402,3 +402,8 @@
[:role [:enum "system" "user" "assistant"]]
[:metadata {:optional true} :any]
[:parts [:sequential {:min 1} ::ui-message-part]]])
+
+(mr/def ::message-data
+ "A whole `metabot_message.data` value in the v2 format: the at-rest projection of a
+ message's `UIMessagePart`s. Assistant placeholder rows are `[]`."
+ [:sequential ::ui-message-part])
diff --git a/test/metabase/metabot/schema/migrate_v1_to_v2_test.clj b/test/metabase/metabot/schema/migrate_v1_to_v2_test.clj
new file mode 100644
index 00000000000..111288a1be0
--- /dev/null
+++ b/test/metabase/metabot/schema/migrate_v1_to_v2_test.clj
@@ -0,0 +1,152 @@
+(ns metabase.metabot.schema.migrate-v1-to-v2-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.metabot.schema.migrate-v1-to-v2 :as migrate]
+ [metabase.metabot.schema.v2 :as schema.v2]
+ [metabase.util.malli.registry :as mr]))
+
+(deftest ^:parallel migrate-v1-external-ai-service->v2-test
+ (testing "TEXT becomes a text part"
+ (is (= [{:type "text" :text "hello"}]
+ (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TEXT" :content "hello"}]))))
+ (testing "batched TOOL_CALL entries split into one part per call, merged with TOOL_RESULTs in position"
+ (is (= [{:type "text" :text "before"}
+ {:type "tool-search" :toolCallId "a" :state "output-available" :input {:q "x"} :output {:output "r1"}}
+ {:type "tool-browse" :toolCallId "b" :state "output-available" :input {} :output {:output "r2"}}
+ {:type "text" :text "after"}]
+ (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TEXT" :content "before"}
+ {:role "assistant" :_type "TOOL_CALL"
+ :tool_calls [{:id "a" :name "search" :arguments "{\"q\":\"x\"}"}
+ {:id "b" :name "browse" :arguments "{}"}]}
+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "a" :content "r1"}
+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "b" :content "r2"}
+ {:role "assistant" :_type "TEXT" :content "after"}]))))
+ (testing "tool name underscores are preserved in :type"
+ (is (= "tool-create_sql_query"
+ (-> (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TOOL_CALL"
+ :tool_calls [{:id "tc1" :name "create_sql_query" :arguments "{}"}]}])
+ first :type))))
+ (testing "TOOL_CALL without a matching result stays input-available with no :output"
+ (let [[part] (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TOOL_CALL"
+ :tool_calls [{:id "tc1" :name "search" :arguments "{}"}]}])]
+ (is (= "input-available" (:state part)))
+ (is (not (contains? part :output)))))
+ (testing "malformed :arguments JSON falls back to the raw string"
+ (is (= "not-json"
+ (-> (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TOOL_CALL"
+ :tool_calls [{:id "tc1" :name "search" :arguments "not-json"}]}])
+ first :input))))
+ (testing "orphan TOOL_RESULT is dropped"
+ (is (= [] (migrate/migrate-v1-external-ai-service->v2
+ [{:role "tool" :_type "TOOL_RESULT" :tool_call_id "nope" :content "x"}]))))
+ (testing "DATA becomes a data-<type> part, dropping :version"
+ (is (= [{:type "data-navigate_to" :data "/metric/17991"}]
+ (migrate/migrate-v1-external-ai-service->v2
+ [{:_type "DATA" :type "navigate_to" :version 1 :value "/metric/17991"}]))))
+ (testing "FINISH_MESSAGE and ERROR are dropped"
+ (is (= [{:type "text" :text "hi"}]
+ (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "TEXT" :content "hi"}
+ {:role "assistant" :_type "ERROR" :content "upstream provider error"}
+ {:role "assistant" :_type "FINISH_MESSAGE" :finish_reason "stop" :usage {}}]))))
+ (testing "unknown :_type throws"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 ai-service entry type"
+ (migrate/migrate-v1-external-ai-service->v2
+ [{:role "assistant" :_type "REASONING" :content "hmm"}])))))
+
+(deftest ^:parallel migrate-v1-native->v2-test
+ (let [tool-input {:type "tool-input" :id "tc1" :function "search" :arguments {:q "x"}}]
+ (testing "tool-input merged with tool-output becomes output-available carrying the result map"
+ (is (= [{:type "tool-search" :toolCallId "tc1" :state "output-available" :input {:q "x"} :output {:output "rows"}}]
+ (migrate/migrate-v1-native->v2
+ [tool-input
+ {:type "tool-output" :id "tc1" :result {:output "rows"} :error nil :duration-ms 3}]))))
+ (testing "empty result map still produces the :output key"
+ (let [[part] (migrate/migrate-v1-native->v2
+ [tool-input {:type "tool-output" :id "tc1" :result {}}])]
+ (is (= "output-available" (:state part)))
+ (is (contains? part :output))
+ (is (= {} (:output part)))))
+ (testing "tool-output with a string :error becomes output-error with :errorText and no :output"
+ (let [[part] (migrate/migrate-v1-native->v2
+ [tool-input {:type "tool-output" :id "tc1" :result {} :error "boom"}])]
+ (is (= "output-error" (:state part)))
+ (is (= "boom" (:errorText part)))
+ (is (not (contains? part :output)))))
+ (testing "map-shaped :error extracts :message"
+ (is (= "bad"
+ (-> (migrate/migrate-v1-native->v2
+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:message "bad"}}])
+ first :errorText))))
+ (testing "map-shaped :error without :message falls back to pr-str"
+ (is (= "{:code 500}"
+ (-> (migrate/migrate-v1-native->v2
+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:code 500}}])
+ first :errorText))))
+ (testing "map-shaped :error with a non-string :message falls back to pr-str"
+ (is (= "{:message {:code 500}}"
+ (-> (migrate/migrate-v1-native->v2
+ [tool-input {:type "tool-output" :id "tc1" :result nil :error {:message {:code 500}}}])
+ first :errorText))))
+ (testing "tool-input without a matching output stays input-available"
+ (let [[part] (migrate/migrate-v1-native->v2 [tool-input])]
+ (is (= "input-available" (:state part)))
+ (is (not (contains? part :output)))
+ (is (not (contains? part :errorText))))))
+ (testing "text passes through with :id stripped"
+ (is (= [{:type "text" :text "Found it"}]
+ (migrate/migrate-v1-native->v2
+ [{:type "text" :id "t1" :text "Found it"}]))))
+ (testing "data entries convert like external DATA entries"
+ (is (= [{:type "data-navigate_to" :data "/question/1"}]
+ (migrate/migrate-v1-native->v2
+ [{:type "data" :data-type "navigate_to" :version 1 :data "/question/1"}]))))
+ (testing "error entries are dropped"
+ (is (= [] (migrate/migrate-v1-native->v2
+ [{:type "error" :error {:message "boom"}}]))))
+ (testing "unknown :type throws"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 native entry type"
+ (migrate/migrate-v1-native->v2
+ [{:type "reasoning" :text "hmm"}])))))
+
+(deftest ^:parallel migrate-v1-user-message->v2-test
+ (testing "user messages convert to text parts"
+ (is (= [{:type "text" :text "Do we have data on orders"}]
+ (migrate/migrate-v1-user-message->v2
+ [{:role "user" :content "Do we have data on orders"}])))))
+
+(deftest ^:parallel migrate-v1->v2-invalid-data-test
+ (testing "a row resembling a v1 shape but failing its schema throws, explaining against the format it resembles"
+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unrecognized v1 storage format"
+ (migrate/migrate-v1->v2 [{:type "text"}])))]
+ (is (= [{:text ["missing required key"]}]
+ (:explanation (ex-data e)))))))
+
+(deftest ^:parallel migrated-rows-validate-test
+ (testing "migrating every v1 row shape should produce valid v2 message data"
+ ;; one row per v1 storage format — ai-service, native, user message, and empty —
+ ;; collectively exercising every entry type a format can hold
+ (let [v1-rows [[{:role "assistant" :_type "TEXT" :content "hello"}
+ {:role "assistant" :_type "TOOL_CALL"
+ :tool_calls [{:id "a" :name "create_sql_query" :arguments "{\"q\":\"x\"}"}
+ {:id "b" :name "browse" :arguments "{}"}]}
+ {:role "tool" :_type "TOOL_RESULT" :tool_call_id "a" :content "r1"}
+ {:_type "DATA" :type "navigate_to" :version 1 :value "/q/1"}
+ {:role "assistant" :_type "FINISH_MESSAGE" :finish_reason "stop" :usage {}}]
+ [{:type "tool-input" :id "tc1" :function "search" :arguments {:q "x"}}
+ {:type "tool-output" :id "tc1" :result {:output "rows"} :error nil :duration-ms 3}
+ {:type "tool-input" :id "tc2" :function "browse" :arguments {}}
+ {:type "tool-output" :id "tc2" :result {} :error {:message "boom"}}
+ {:type "tool-input" :id "tc3" :function "search" :arguments {:q "y"}}
+ {:type "text" :id "t1" :text "Found it"}
+ {:type "data" :data-type "state" :version 1 :data {}}
+ {:type "error" :error {:message "boom"}}]
+ [{:role "user" :content "Do we have data on orders"}]
+ []]]
+ (is (nil? (mr/explain [:sequential ::schema.v2/message-data]
+ (mapv migrate/migrate-v1->v2 v1-rows)))))))