The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[Metabot] Handle `content-filter` finish reason (#79336)
Sloan Spargera33d96bbd3a1
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsxindex d889e52471a..b4f7d967219 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx@@ -500,7 +500,7 @@ const IncompleteTurnAlert = ({continuable: true,})).with("content-filter", () => ({- message: t`Response from ${metabotName} was stopped by a content filter`,+ message: t`Response from ${metabotName} was stopped by a content filter. Try rephrasing your question.`,continuable: false,})).with("tool-calls", "other", () => ({diff --git a/frontend/src/metabase/metabot/tests/message.unit.spec.tsx b/frontend/src/metabase/metabot/tests/message.unit.spec.tsxindex 7d42c1bb076..3fe04ec5bb6 100644--- a/frontend/src/metabase/metabot/tests/message.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/message.unit.spec.tsx@@ -118,74 +118,99 @@ const truncatedResponse: SSEEvent[] = [{ type: "finish", finishReason: "length" },];-describe("metabot > truncated responses", () => {- it("should keep the partial text and offer to continue", async () => {- setup();- mockAgentEndpoint({ events: truncatedResponse });-- await enterChatMessage("Tell me everything");-- await assertConversation([- ["user", "Tell me everything"],- ["agent", "Here is the start of a long answer"],- ["agent", /was cut off/],- ]);- expect(await continueResponseButton()).toBeInTheDocument();- });-- it("should send a continuation user turn when continue is clicked", async () => {- setup();- mockAgentEndpoint({ events: truncatedResponse });- await enterChatMessage("Tell me everything");-- const continuationSpy = mockAgentEndpoint({- events: [- { type: "start", messageId: "msg_continued" },- { type: "text-start", id: "t2" },- { type: "text-delta", id: "t2", delta: "and here is the rest." },- { type: "text-end", id: "t2" },- { type: "finish", finishReason: "stop" },- ],+describe("metabot > finish reason", () => {+ describe("length", () => {+ it("should keep the partial text and offer to continue", async () => {+ setup();+ mockAgentEndpoint({ events: truncatedResponse });++ await enterChatMessage("Tell me everything");++ await assertConversation([+ ["user", "Tell me everything"],+ ["agent", "Here is the start of a long answer"],+ ["agent", /was cut off/],+ ]);+ expect(await continueResponseButton()).toBeInTheDocument();});- await userEvent.click(await continueResponseButton());+ it("should send a continuation user turn when continue is clicked", async () => {+ setup();+ mockAgentEndpoint({ events: truncatedResponse });+ await enterChatMessage("Tell me everything");++ const continuationSpy = mockAgentEndpoint({+ events: [+ { type: "start", messageId: "msg_continued" },+ { type: "text-start", id: "t2" },+ { type: "text-delta", id: "t2", delta: "and here is the rest." },+ { type: "text-end", id: "t2" },+ { type: "finish", finishReason: "stop" },+ ],+ });++ await userEvent.click(await continueResponseButton());++ await assertConversation([+ ["user", "Tell me everything"],+ ["agent", "Here is the start of a long answer"],+ ["agent", /was cut off/],+ ["user", /Your last response was cut off/],+ ["agent", "and here is the rest."],+ ]);+ expect((await lastReqBody(continuationSpy))?.message).toMatch(+ /Pick up exactly where you left off/,+ );+ });- await assertConversation([- ["user", "Tell me everything"],- ["agent", "Here is the start of a long answer"],- ["agent", /was cut off/],- ["user", /Your last response was cut off/],- ["agent", "and here is the rest."],- ]);- expect((await lastReqBody(continuationSpy))?.message).toMatch(- /Pick up exactly where you left off/,- );- });+ it("should not offer continue on earlier turns", async () => {+ setup();+ mockAgentEndpoint({ events: truncatedResponse });+ await enterChatMessage("Tell me everything");+ expect(await continueResponseButton()).toBeInTheDocument();- it("should not offer continue on earlier turns", async () => {- setup();- mockAgentEndpoint({ events: truncatedResponse });- await enterChatMessage("Tell me everything");- expect(await continueResponseButton()).toBeInTheDocument();+ mockAgentEndpoint({ events: whoIsYourFavoriteResponse });+ await enterChatMessage("Nevermind, who is your favorite?");+ expect(+ await screen.findByText("You, but don't tell anyone."),+ ).toBeInTheDocument();- mockAgentEndpoint({ events: whoIsYourFavoriteResponse });- await enterChatMessage("Nevermind, who is your favorite?");- expect(- await screen.findByText("You, but don't tell anyone."),- ).toBeInTheDocument();-- expect(queryContinueResponseButton()).not.toBeInTheDocument();+ expect(queryContinueResponseButton()).not.toBeInTheDocument();+ });});- it("should not show a notice for a normal stop", async () => {- setup();- mockAgentEndpoint({ events: whoIsYourFavoriteResponse });+ describe("content-filter", () => {+ it("should explain a content-filtered response", async () => {+ setup();+ mockAgentEndpoint({+ events: [+ { type: "start", messageId: "msg_filtered" },+ { type: "text-start", id: "t1" },+ { type: "text-delta", id: "t1", delta: "I can't help with that." },+ { type: "text-end", id: "t1" },+ { type: "finish", finishReason: "content-filter" },+ ],+ });++ await enterChatMessage("Tell me everything");++ await assertConversation([+ ["user", "Tell me everything"],+ ["agent", "I can't help with that."],+ ["agent", /stopped by a content filter/],+ ]);+ });- await enterChatMessage("Who is your favorite?");- await assertConversation([- ["user", "Who is your favorite?"],- ["agent", "You, but don't tell anyone."],- ]);- expect(queryTurnAlert()).not.toBeInTheDocument();+ it("should not show a notice for a normal stop", async () => {+ setup();+ mockAgentEndpoint({ events: whoIsYourFavoriteResponse });++ await enterChatMessage("Who is your favorite?");+ await assertConversation([+ ["user", "Who is your favorite?"],+ ["agent", "You, but don't tell anyone."],+ ]);+ expect(queryTurnAlert()).not.toBeInTheDocument();+ });});});diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex fd1a034a2d1..e9c3018b678 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -424,9 +424,10 @@(rf (format-sse-event(cond-> {:type "finish":finishReason (cond- (= @finish-reason "length") "length"- @error? "error"- :else "stop")}+ (= @finish-reason "length") "length"+ @error? "error"+ (= @finish-reason "content-filter") "content-filter"+ :else "stop")}(seq metadata) (assoc :messageMetadata metadata))))(rf done-sse-line)(rf))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 6dfd32c12dd..96c78be0544 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -613,7 +613,17 @@(is (=? {:type "finish" :finishReason "length"}(last (sse-events [{:type :error :error {:message "tool blew up mid-turn"}}{:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}- :finish-reason "length" :raw-finish-reason "max_tokens"}]))))))+ :finish-reason "length" :raw-finish-reason "max_tokens"}])))))+ (testing "a filtered turn emits finishReason \"content-filter\""+ (is (=? {:type "finish" :finishReason "content-filter"}+ (last (sse-events [{:type :text :id "t1" :text "I can't help with that."}+ {:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}+ :finish-reason "content-filter" :raw-finish-reason "refusal"}])))))+ (testing "an in-turn error part outranks a content-filter finish — the errored turn is rewound"+ (is (=? {:type "finish" :finishReason "error"}+ (last (sse-events [{:type :error :error {:message "tool blew up mid-turn"}}+ {:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}+ :finish-reason "content-filter" :raw-finish-reason "refusal"}]))))))(deftest parts->aisdk-sse-xf-lifecycle-test(testing "first :start opens the message and a step; later :start is a step boundary; completion closes"