Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(web): chat messages keep role titles on non-generation observations (#19259)

Trang Nguyen943adbb5c0c9
diff --git a/packages/shared/scripts/seeder/scenarios/timeline-shapes.ts b/packages/shared/scripts/seeder/scenarios/timeline-shapes.ts
index 5f18921271..453a0ea07d 100644
--- a/packages/shared/scripts/seeder/scenarios/timeline-shapes.ts
+++ b/packages/shared/scripts/seeder/scenarios/timeline-shapes.ts
@@ -171,6 +171,13 @@ const SHAPES: Shape[] = [
name: "chat.turn",
start: 0,
end: 8180,
+ // Chat transcript on a SPAN: the preview must keep role titles here.
+ input: [
+ { role: "user", content: "hello" },
+ { role: "assistant", content: "hi, how can I help?" },
+ { role: "user", content: "a nice car" },
+ ],
+ output: [{ role: "assistant", content: "sure" }],
},
{
key: "guard",
@@ -191,6 +198,15 @@ const SHAPES: Shape[] = [
model: "gpt-5.4",
usage: [520, 1480],
ttft: 640,
+ input: {
+ messages: [
+ { role: "system", content: "You are a car-buying assistant." },
+ { role: "user", content: "hello" },
+ { role: "assistant", content: "hi, how can I help?" },
+ { role: "user", content: "a nice car" },
+ ],
+ },
+ output: { role: "assistant", content: "sure" },
},
{
key: "persist",
diff --git a/web/src/features/prompts/components/PromptDetail.tsx b/web/src/features/prompts/components/PromptDetail.tsx
index 3bc1961fa0..4655c1b759 100644
--- a/web/src/features/prompts/components/PromptDetail.tsx
+++ b/web/src/features/prompts/components/PromptDetail.tsx
@@ -8,7 +8,7 @@ import {
withDefault,
} from "use-query-params";
import type { z } from "zod";
-import { ChatMessageList } from "@/src/features/traces";
+import { ChatMessageList, getMessageTitle } from "@/src/features/traces";
import { Tabs } from "@/src/components/design-system/Tabs/Tabs";
import { BadgeShell } from "@/src/components/design-system/Badge/Badge";
import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
@@ -727,6 +727,7 @@ export const PromptDetail = ({
currentView="pretty"
messageToToolCallNumbers={new Map()}
collapseLongHistory={false}
+ getMessageTitle={getMessageTitle}
/>
</div>
) : typeof prompt.prompt === "string" ? (
diff --git a/web/src/features/traces/components/ChatMessageList.clienttest.tsx b/web/src/features/traces/components/ChatMessageList.clienttest.tsx
index ee77928ca3..ded6566bd3 100644
--- a/web/src/features/traces/components/ChatMessageList.clienttest.tsx
+++ b/web/src/features/traces/components/ChatMessageList.clienttest.tsx
@@ -34,7 +34,10 @@ vi.mock(
);
import { ChatMessageList } from "@/src/features/traces/components/ChatMessageList";
-import { type ChatMlMessage } from "@/src/features/traces/fns/chatMessageUtils";
+import {
+ type ChatMlMessage,
+ getMessageTitle,
+} from "@/src/features/traces/fns/chatMessageUtils";
import { type MediaReturnType } from "@/src/features/media";
// Pin the limit so the test does not depend on the ambient .env value.
const { TEST_LIMIT } = vi.hoisted(() => ({ TEST_LIMIT: 1_000 }));
@@ -53,6 +56,7 @@ const renderList = (message: ChatMlMessage, mediaIds: string[]) =>
media={mediaIds.map((mediaId) => ({ mediaId }) as MediaReturnType)}
currentView="pretty"
messageToToolCallNumbers={new Map()}
+ getMessageTitle={getMessageTitle}
/>,
);
@@ -78,6 +82,7 @@ describe("ChatMessageList long history", () => {
shouldRenderMarkdown={true}
currentView="pretty"
messageToToolCallNumbers={new Map()}
+ getMessageTitle={getMessageTitle}
/>,
);
@@ -107,6 +112,7 @@ describe("ChatMessageList long history", () => {
shouldRenderMarkdown={true}
currentView="pretty"
messageToToolCallNumbers={new Map()}
+ getMessageTitle={getMessageTitle}
/>,
);
@@ -176,6 +182,7 @@ describe("ChatMessageList long history", () => {
shouldRenderMarkdown={true}
currentView="pretty"
messageToToolCallNumbers={new Map()}
+ getMessageTitle={getMessageTitle}
collapseLongHistory={false}
/>,
);
diff --git a/web/src/features/traces/components/ChatMessageList.tsx b/web/src/features/traces/components/ChatMessageList.tsx
index 1feefb854d..e78728cd7d 100644
--- a/web/src/features/traces/components/ChatMessageList.tsx
+++ b/web/src/features/traces/components/ChatMessageList.tsx
@@ -15,7 +15,6 @@ import {
import { SectionMedia } from "@/src/features/traces/components/IOPreview/components/SectionMedia";
import {
type ChatMlMessage,
- getMessageTitle,
shouldRenderMessageForContentMode,
} from "@/src/features/traces/fns/chatMessageUtils";
import { type MediaReturnType } from "@/src/features/media";
@@ -44,8 +43,8 @@ export interface ChatMessageListProps {
inputMessageCount?: number;
contentMode?: IOPreviewContentMode;
showSystemPrompt?: boolean;
- /** Title messages Input / Output by side instead of by role. */
- titleBySide?: boolean;
+ /** Section title per message; the caller owns the rule. */
+ getMessageTitle: (message: ChatMlMessage, side: "input" | "output") => string;
/** Header actions for the last output-side message (e.g. Correct output). */
outputActions?: IOSectionAction[];
}
@@ -72,7 +71,7 @@ export function ChatMessageList({
inputMessageCount,
contentMode = "all",
showSystemPrompt,
- titleBySide = false,
+ getMessageTitle,
outputActions,
}: ChatMessageListProps) {
// Filter messages to only those with renderable content
@@ -142,7 +141,6 @@ export function ChatMessageList({
<div className="flex flex-col gap-2">
{messagesToRender.map(({ message, originalIndex }, filteredIndex) => {
const isOutputMessage = originalIndex >= (inputMessageCount ?? 0);
- const sideTitle = isOutputMessage ? "Output" : "Input";
const folded = isFolded(filteredIndex);
// Folded messages do not mount (no parsing) until the row opens.
if (folded && !othersOpen) return null;
@@ -150,7 +148,10 @@ export function ChatMessageList({
<ChatMessage
key={originalIndex}
message={message}
- title={titleBySide ? sideTitle : getMessageTitle(message)}
+ title={getMessageTitle(
+ message,
+ isOutputMessage ? "output" : "input",
+ )}
shouldRenderMarkdown={shouldRenderMarkdown}
currentView={currentView}
toolCallNumbers={messageToToolCallNumbers.get(originalIndex)}
diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.stories.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.stories.tsx
index bb5282d15a..c504cb3684 100644
--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.stories.tsx
+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.stories.tsx
@@ -183,6 +183,42 @@ export const Loading = meta.story({
},
});
+export const SpanWithChatMessages = meta.story({
+ name: "(Test) Span keeps role titles",
+ args: {
+ observationType: "SPAN",
+ input: [
+ { role: "user", content: "hello" },
+ { role: "assistant", content: "hi, how can I help?" },
+ { role: "user", content: "a nice car" },
+ ],
+ output: [{ role: "assistant", content: "sure" }],
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ // Each message mounts a formatted and a hidden JSON header.
+ await expect(canvas.getAllByText("user").length).toBeGreaterThan(0);
+ await expect(canvas.getAllByText("assistant").length).toBeGreaterThan(0);
+ await expect(canvas.queryAllByText("Input")).toHaveLength(0);
+ await expect(canvas.queryAllByText("Output")).toHaveLength(0);
+ },
+});
+
+export const SpanWithPlainText = meta.story({
+ name: "(Test) Span titles plain text by side",
+ args: {
+ observationType: "SPAN",
+ input: "Summarize this support conversation.",
+ output: "The customer needs help configuring prompt caching.",
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ await expect(canvas.getAllByText("Input").length).toBeGreaterThan(0);
+ await expect(canvas.getAllByText("Output").length).toBeGreaterThan(0);
+ await expect(canvas.queryAllByText("user")).toHaveLength(0);
+ },
+});
+
export const Empty = meta.story({
name: "(Test) Empty",
args: { input: undefined, output: undefined },
diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
index b2accb6188..03896096df 100644
--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
@@ -16,7 +16,7 @@ import {
useIOPreviewParser,
} from "../../hooks/useIOPreviewParser";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
-import { ChatMessageList } from "../ChatMessageList";
+import { ChatMessageList, type ChatMessageListProps } from "../ChatMessageList";
import { SectionToolDefinitions } from "./components/SectionToolDefinitions";
import {
SectionSkills,
@@ -31,7 +31,10 @@ import {
useCorrectOutputAction,
} from "./components/CorrectedOutputField";
import { useCorrectionData } from "./hooks/useCorrectionData";
-import { shouldRenderMessageForContentMode } from "@/src/features/traces/fns/chatMessageUtils";
+import {
+ getMessageTitle,
+ shouldRenderMessageForContentMode,
+} from "@/src/features/traces/fns/chatMessageUtils";
import { StatusMessageSection } from "./components/StatusMessageSection";
import type { ObservationStatusMessage } from "./components/statusMessagePresentation";
@@ -105,6 +108,30 @@ function JsonInputOutputView({
);
}
+function hasDeclaredRole(value: unknown): boolean {
+ return (
+ value !== null &&
+ typeof value === "object" &&
+ typeof (value as { role?: unknown }).role === "string"
+ );
+}
+
+/**
+ * Chat I/O: a message array, bare or under `messages`, or a single message,
+ * with at least one declared string `role`. Anything else is non-chat I/O
+ * that the preview wraps into messages itself.
+ */
+function isChatIO(value: unknown): boolean {
+ const messages =
+ value !== null && typeof value === "object" && !Array.isArray(value)
+ ? (value as { messages?: unknown }).messages
+ : value;
+ if (Array.isArray(messages)) return messages.some(hasDeclaredRole);
+ return hasDeclaredRole(value);
+}
+
+const SIDE_TITLES = { input: "Input", output: "Output" } as const;
+
export interface IOPreviewPrettyProps extends ExpansionStateProps {
skills?: SectionSkillsProps["skills"];
input?: Prisma.JsonValue;
@@ -337,11 +364,19 @@ export function IOPreviewPretty({
onClosed={() => setCorrectionOpen(false)}
/>
) : null;
- // Only generations hold a conversation; tools, spans, agents and the rest
- // are titled Input / Output rather than by role. (`isGenerationLike` is
- // wider: it also covers TOOL and AGENT, which carry model usage, not chat.)
- const titleBySide =
+ // Non-generation I/O that is not chat (plain strings, loose records) is
+ // titled Input / Output; declared chat roles keep their role titles on
+ // every observation type. Generations always title by role.
+ const canTitleBySide =
observationType !== undefined && observationType !== "GENERATION";
+ const titleBySide = {
+ input: canTitleBySide && !isChatIO(parsedInput),
+ output: canTitleBySide && !isChatIO(parsedOutput),
+ };
+ const messageTitle: ChatMessageListProps["getMessageTitle"] = (
+ message,
+ side,
+ ) => (titleBySide[side] ? SIDE_TITLES[side] : getMessageTitle(message));
return (
<div className="space-y-2 pt-3">
@@ -377,7 +412,7 @@ export function IOPreviewPretty({
inputMessageCount={inputMessageCount}
contentMode={contentMode}
showSystemPrompt={showSystemPrompt}
- titleBySide={titleBySide}
+ getMessageTitle={messageTitle}
outputActions={outputActions}
/>
{correctedOutputField}
diff --git a/web/src/features/traces/index.ts b/web/src/features/traces/index.ts
index 947dd90c6d..6208408d36 100644
--- a/web/src/features/traces/index.ts
+++ b/web/src/features/traces/index.ts
@@ -27,6 +27,7 @@ export {
export { ChatMessageList } from "@/src/features/traces/components/ChatMessageList";
export { ThinkingBlock } from "@/src/features/traces/components/ThinkingBlock";
export {
+ getMessageTitle,
hasRenderableConversationMessages,
isOnlyJsonMessage,
} from "@/src/features/traces/fns/chatMessageUtils";