Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metabot /debug mode to show tool calls (#64811)

Noah Moss525dec7eb086
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotAgentToolCallMessage.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotAgentToolCallMessage.tsx
new file mode 100644
index 00000000000..81af983f339
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotAgentToolCallMessage.tsx
@@ -0,0 +1,128 @@
+import { useClipboard, useDisclosure } from "@mantine/hooks";
+import { useMemo } from "react";
+import { t } from "ttag";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+import {
+ ActionIcon,
+ Badge,
+ Box,
+ Button,
+ Flex,
+ Icon,
+ Modal,
+ Stack,
+ Text,
+} from "metabase/ui";
+import type { MetabotDebugToolCallMessage } from "metabase-enterprise/metabot/state";
+
+const ToolCallDetailsModal = ({
+ message,
+ onClose,
+}: {
+ message: MetabotDebugToolCallMessage;
+ onClose: () => void;
+}) => {
+ const clipboard = useClipboard();
+ const copy = (value: any) => clipboard.copy(JSON.stringify(value, null, 2));
+
+ const parsedArgs = useMemo(() => {
+ try {
+ return message.args
+ ? // done for formatting
+ JSON.stringify(JSON.parse(message.args), null, 2)
+ : "{}";
+ } catch {
+ console.warn("Failed to parse tool call args as JSON", message.args);
+ return message.args ?? "{}";
+ }
+ }, [message.args]);
+
+ return (
+ <Modal
+ opened
+ onClose={onClose}
+ size="lg"
+ title={
+ <Flex align="center">
+ {t`Tool Call: ${message.name}`}
+ <Badge ml="sm" color="text-dark">
+ {message.id}
+ </Badge>
+ </Flex>
+ }
+ data-testid="tool-call-details-modal"
+ >
+ <Stack gap="md">
+ {message.args && (
+ <Stack gap="xs">
+ <Flex gap="xs">
+ <Text fw="bold">{t`Request`}</Text>
+ <ActionIcon h="sm" onClick={() => copy(parsedArgs)}>
+ <Icon name="copy" size="1rem" />
+ </ActionIcon>
+ </Flex>
+ <Box mx="-1.5rem">
+ <CodeEditor value={parsedArgs} language="json" readOnly />
+ </Box>
+ </Stack>
+ )}
+
+ {message.result && (
+ <Stack gap="xs">
+ <Flex gap="xs">
+ <Text fw="bold">{t`Response`}</Text>
+ <ActionIcon h="sm" onClick={() => copy(message.result)}>
+ <Icon name="copy" size="1rem" />
+ </ActionIcon>
+ </Flex>
+ <Box mx="-1.5rem">
+ <CodeEditor value={message.result} readOnly />
+ </Box>
+ </Stack>
+ )}
+ </Stack>
+ </Modal>
+ );
+};
+
+export const AgentToolCallMessage = ({
+ message,
+}: {
+ message: MetabotDebugToolCallMessage;
+}) => {
+ const [modalOpen, { open, close }] = useDisclosure(false);
+ const clipboard = useClipboard();
+ const handleCopy = () => clipboard.copy(JSON.stringify(message, null, 2));
+
+ return (
+ <>
+ <Flex
+ p="sm"
+ pl="md"
+ bg="bg-light"
+ bd="1px solid var(--mb-color-border)"
+ bdrs="sm"
+ direction="row"
+ align="center"
+ justify="space-between"
+ >
+ <Flex>
+ <Text mr="sm">🔧</Text>
+ <Text fw="bold">{message.name}</Text>
+ </Flex>
+ <Flex align="center" gap="xs">
+ <Button
+ variant="light"
+ size="compact-xs"
+ onClick={open}
+ >{t`View`}</Button>
+ <ActionIcon h="sm" onClick={handleCopy}>
+ <Icon name="copy" size="1rem" />
+ </ActionIcon>
+ </Flex>
+ </Flex>
+ {modalOpen && <ToolCallDetailsModal message={message} onClose={close} />}
+ </>
+ );
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChat.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChat.tsx
index d358da3aaa9..c016a51cbf7 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChat.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChat.tsx
@@ -177,7 +177,7 @@ export const MetabotChat = ({
/>
{/* loading */}
{metabot.isDoingScience && (
- <MetabotThinking toolCalls={metabot.toolCalls} />
+ <MetabotThinking toolCalls={metabot.activeToolCalls} />
)}
{/* filler - height gets set via ref mutation */}
<div ref={fillerRef} data-testid="metabot-message-filler" />
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChatMessage.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChatMessage.tsx
index f89957f0315..40b2be369d6 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChatMessage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotChat/MetabotChatMessage.tsx
@@ -26,6 +26,7 @@ import { AIMarkdown } from "../AIMarkdown/AIMarkdown";
import { AgentSuggestionMessage } from "./MetabotAgentSuggestionMessage";
import { AgentTodoListMessage } from "./MetabotAgentTodoMessage";
+import { AgentToolCallMessage } from "./MetabotAgentToolCallMessage";
import Styles from "./MetabotChat.module.css";
import { MetabotFeedbackModal } from "./MetabotFeedbackModal";
@@ -157,6 +158,9 @@ export const AgentMessage = ({
{message.type === "todo_list" && (
<AgentTodoListMessage todos={message.payload} />
)}
+ {message.type === "tool_call" && (
+ <AgentToolCallMessage message={message} />
+ )}
<Flex className={Styles.messageActions}>
{!hideActions && (
<>
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/hooks/use-metabot-agent.ts b/enterprise/frontend/src/metabase-enterprise/metabot/hooks/use-metabot-agent.ts
index 52bcc38aee9..00b2ffd6bbb 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/hooks/use-metabot-agent.ts
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/hooks/use-metabot-agent.ts
@@ -8,7 +8,9 @@ import { trackMetabotRequestSent } from "../analytics";
import {
type MetabotPromptSubmissionResult,
type MetabotUserChatMessage,
+ getActiveToolCalls,
getAgentErrorMessages,
+ getDebugMode,
getIsLongMetabotConversation,
getIsProcessing,
getMessages,
@@ -17,7 +19,6 @@ import {
getMetabotRequestId,
getMetabotVisible,
getProfileOverride,
- getToolCalls,
resetConversation as resetConversationAction,
retryPrompt,
setVisible as setVisibleAction,
@@ -57,8 +58,12 @@ export const useMetabotAgent = () => {
getMetabotRequestId as any,
) as ReturnType<typeof getMetabotRequestId>;
- const toolCalls = useSelector(getToolCalls as any) as ReturnType<
- typeof getToolCalls
+ const activeToolCalls = useSelector(getActiveToolCalls as any) as ReturnType<
+ typeof getActiveToolCalls
+ >;
+
+ const debugMode = useSelector(getDebugMode as any) as ReturnType<
+ typeof getDebugMode
>;
const setVisible = useCallback(
@@ -175,7 +180,8 @@ export const useMetabotAgent = () => {
startNewConversation,
submitInput,
retryMessage,
- toolCalls,
+ activeToolCalls,
+ debugMode,
profileOverride,
reactions,
};
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/metabot.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/metabot.unit.spec.tsx
index be331d5962d..3eafc84643a 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/metabot.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/metabot.unit.spec.tsx
@@ -1022,6 +1022,58 @@ describe("metabot-streaming", () => {
expect(screen.queryByText(/xxxxxxx/)).not.toBeInTheDocument();
});
});
+
+ describe("experimental", () => {
+ describe("debug mode", () => {
+ const mockResponse = () => {
+ mockAgentEndpoint(
+ {
+ textChunks: [
+ `0:"Before"`,
+ `9:{"toolCallId":"debug_test","toolName":"debug_test","args":""}`,
+ `a:{"toolCallId":"debug_test","result":""}`,
+ `0:"After"`,
+ `d:{"finishReason":"stop","usage":{"promptTokens":4916,"completionTokens":8}}`,
+ ],
+ }, // small delay to cause loading state
+ );
+ };
+
+ it("should not show tool_call messages in chat if debug mode is disabled", async () => {
+ setup();
+ mockResponse();
+
+ await enterChatMessage("Don't show me tool call messages");
+ await assertConversation([
+ ["user", "Don't show me tool call messages"],
+ ["agent", "Before"],
+ ["agent", "After"],
+ ]);
+ });
+
+ it("should show tool_call messages in chat if debug mode is enabled", async () => {
+ setup({
+ metabotPluginInitialState: {
+ ...getMetabotInitialState(),
+ visible: true,
+ experimental: {
+ ...getMetabotInitialState().experimental,
+ debugMode: true,
+ },
+ },
+ });
+ mockResponse();
+
+ await enterChatMessage("Don't show me tool call messages");
+ await assertConversation([
+ ["user", "Don't show me tool call messages"],
+ ["agent", "Before"],
+ ["agent", "debug_test"],
+ ["agent", "After"],
+ ]);
+ });
+ });
+ });
});
const whoIsYourFavoriteResponse = [
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/state/actions.ts b/enterprise/frontend/src/metabase-enterprise/metabot/state/actions.ts
index cbb5506e914..dd45c8c4849 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/state/actions.ts
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/state/actions.ts
@@ -33,13 +33,14 @@ import {
import {
getAgentErrorMessages,
getAgentRequestMetadata,
+ getDebugMode,
getHistory,
getIsProcessing,
getLastMessage,
getMetabotConversationId,
getUserPromptForMessageId,
} from "./selectors";
-import type { SlashCommand } from "./types";
+import type { MetabotStoreState, SlashCommand } from "./types";
import { createMessageId, parseSlashCommand } from "./utils";
export const {
@@ -54,6 +55,7 @@ export const {
toolCallEnd,
setProfileOverride,
setMetabotReqIdOverride,
+ setDebugMode,
addSuggestedTransform,
activateSuggestedTransform,
deactivateSuggestedTransform,
@@ -105,7 +107,7 @@ export const setVisible =
export const executeSlashCommand = createAsyncThunk<void, SlashCommand>(
"metabase-enterprise/metabot/executeSlashCommand",
- async (slashCommand, { dispatch }) => {
+ async (slashCommand, { dispatch, getState }) => {
match(slashCommand)
.with({ cmd: "profile" }, ({ args }) => {
if (args.length <= 1) {
@@ -121,6 +123,18 @@ export const executeSlashCommand = createAsyncThunk<void, SlashCommand>(
dispatch(addUndo({ message: "/metabot <name>" }));
}
})
+ .with({ cmd: "debug" }, () => {
+ const currentDebugMode = getDebugMode(getState() as MetabotStoreState);
+ const newDebugMode = !currentDebugMode;
+ dispatch(setDebugMode(newDebugMode));
+ dispatch(
+ addUndo({
+ message: newDebugMode
+ ? "Debug mode enabled"
+ : "Debug mode disabled",
+ }),
+ );
+ })
.otherwise(() => {
dispatch(addUndo({ message: "Unknown command" }));
});
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/state/reducer.ts b/enterprise/frontend/src/metabase-enterprise/metabot/state/reducer.ts
index e15a2e9a8c6..cc201517df8 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/state/reducer.ts
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/state/reducer.ts
@@ -55,18 +55,32 @@ export type MetabotAgentEditSuggestionChatMessage = {
};
};
+export type MetabotDebugToolCallMessage = {
+ id: string;
+ role: "agent";
+ type: "tool_call";
+ name: string;
+ args?: string;
+ status: "started" | "ended";
+ result?: string;
+};
+
export type MetabotAgentChatMessage =
| MetabotAgentTextChatMessage
| MetabotAgentTodoListChatMessage
- | MetabotAgentEditSuggestionChatMessage;
+ | MetabotAgentEditSuggestionChatMessage
+ | MetabotDebugToolCallMessage;
export type MetabotUserChatMessage =
| MetabotUserTextChatMessage
| MetabotUserActionChatMessage;
+export type MetabotDebugChatMessage = MetabotDebugToolCallMessage;
+
export type MetabotChatMessage =
| MetabotUserChatMessage
- | MetabotAgentChatMessage;
+ | MetabotAgentChatMessage
+ | MetabotDebugChatMessage;
export type MetabotErrorMessage = {
type: "message" | "alert";
@@ -99,8 +113,9 @@ export interface MetabotState {
history: MetabotHistory;
state: any;
reactions: MetabotReactionsState;
- toolCalls: MetabotToolCall[];
+ activeToolCalls: MetabotToolCall[];
experimental: {
+ debugMode: boolean;
metabotReqIdOverride: string | undefined;
profileOverride: string | undefined;
};
@@ -118,8 +133,9 @@ export const getMetabotInitialState = (): MetabotState => ({
navigateToPath: null,
suggestedTransforms: [],
},
- toolCalls: [],
+ activeToolCalls: [],
experimental: {
+ debugMode: false,
metabotReqIdOverride: undefined,
profileOverride: undefined,
},
@@ -143,7 +159,7 @@ export const metabot = createSlice({
state,
action: PayloadAction<Omit<MetabotAgentChatMessage, "id" | "role">>,
) => {
- state.toolCalls = [];
+ state.activeToolCalls = [];
state.messages.push({
id: createMessageId(),
role: "agent",
@@ -160,7 +176,7 @@ export const metabot = createSlice({
state.errorMessages.push(action.payload);
},
addAgentTextDelta: (state, action: PayloadAction<string>) => {
- const hasToolCalls = state.toolCalls.length > 0;
+ const hasToolCalls = state.activeToolCalls.length > 0;
const lastMessage = _.last(state.messages);
const canAppend =
!hasToolCalls &&
@@ -178,24 +194,49 @@ export const metabot = createSlice({
});
}
- state.toolCalls = hasToolCalls ? [] : state.toolCalls;
+ state.activeToolCalls = hasToolCalls ? [] : state.activeToolCalls;
},
toolCallStart: (
state,
- action: PayloadAction<{ toolCallId: string; toolName: string }>,
+ action: PayloadAction<{
+ toolCallId: string;
+ toolName: string;
+ args?: string;
+ }>,
) => {
- const { toolCallId, toolName } = action.payload;
- state.toolCalls.push({
+ const { toolCallId, toolName, args } = action.payload;
+ state.messages.push({
+ id: toolCallId,
+ role: "agent",
+ type: "tool_call",
+ name: toolName,
+ args,
+ status: "started",
+ });
+ state.activeToolCalls.push({
id: toolCallId,
name: toolName,
message: TOOL_CALL_MESSAGES[toolName],
status: "started",
});
},
- toolCallEnd: (state, action: PayloadAction<{ toolCallId: string }>) => {
- state.toolCalls = state.toolCalls.map((tc) =>
+ toolCallEnd: (
+ state,
+ action: PayloadAction<{ toolCallId: string; result?: any }>,
+ ) => {
+ state.activeToolCalls = state.activeToolCalls.map((tc) =>
tc.id === action.payload.toolCallId ? { ...tc, status: "ended" } : tc,
);
+
+ // Update the message in messages array with result for debug history
+ const message = state.messages.findLast(
+ (msg) =>
+ msg.type === "tool_call" && msg.id === action.payload.toolCallId,
+ );
+ if (message?.type === "tool_call") {
+ message.status = "ended";
+ message.result = action.payload.result;
+ }
},
// NOTE: this reducer fn should be made smarter if/when we want to have
// metabot's `state` object be able to remove / forget values. currently
@@ -220,7 +261,7 @@ export const metabot = createSlice({
state.history = [];
state.state = {};
state.isProcessing = false;
- state.toolCalls = [];
+ state.activeToolCalls = [];
state.conversationId = uuid();
state.reactions.suggestedTransforms = [];
state.experimental.metabotReqIdOverride = undefined;
@@ -246,6 +287,9 @@ export const metabot = createSlice({
setProfileOverride: (state, action: PayloadAction<string | undefined>) => {
state.experimental.profileOverride = action.payload;
},
+ setDebugMode: (state, action: PayloadAction<boolean>) => {
+ state.experimental.debugMode = action.payload;
+ },
addSuggestedTransform: (
state,
{ payload: transform }: PayloadAction<MetabotSuggestedTransform>,
@@ -297,11 +341,11 @@ export const metabot = createSlice({
.addCase(sendAgentRequest.fulfilled, (state, action) => {
state.history = action.payload?.history?.slice() ?? [];
state.state = { ...(action.payload?.state ?? {}) };
- state.toolCalls = [];
+ state.activeToolCalls = [];
state.isProcessing = false;
})
.addCase(sendAgentRequest.rejected, (state) => {
- state.toolCalls = [];
+ state.activeToolCalls = [];
state.isProcessing = false;
});
},
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/state/selectors.ts b/enterprise/frontend/src/metabase-enterprise/metabot/state/selectors.ts
index 0f6a9369046..67fc7bfbba9 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/state/selectors.ts
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/state/selectors.ts
@@ -24,14 +24,28 @@ export const getMetabotVisible = createSelector(
(metabot) => metabot.visible,
);
-export const getMessages = createSelector(
+export const getDebugMode = createSelector(
+ getMetabot,
+ (metabot) => metabot.experimental.debugMode,
+);
+
+const getInternalMessages = createSelector(
getMetabot,
(metabot) => metabot.messages,
);
-export const getToolCalls = createSelector(
+export const getMessages = createSelector(
+ [getInternalMessages, getDebugMode],
+ (messages, debugMode) => {
+ return debugMode
+ ? messages
+ : messages.filter((msg) => msg.type !== "tool_call");
+ },
+);
+
+export const getActiveToolCalls = createSelector(
getMetabot,
- (metabot) => metabot.toolCalls,
+ (metabot) => metabot.activeToolCalls,
);
export const getLastMessage = createSelector(getMessages, (messages) =>