🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(sessions): Add experimental root only mode for session transcripts (#19145)
Ben Bachem08457e2198d1
diff --git a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.tsindex fa1cf35e79..97816f24f9 100644--- a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts+++ b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts@@ -360,6 +360,65 @@ it("returns the shared transcript unchanged without fetching root I/O", async ()expect(mocks.root).not.toHaveBeenCalled();});+it("uses only root I/O in root mode even when generations exist", async () => {+ const root = {+ id: "root",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: "Root question",+ output: "Root answer",+ metadata: {},+ };+ mocks.observations.mockResolvedValue({+ observations: [root, { id: "generation", type: "GENERATION" }],+ totalCount: 2,+ });+ mocks.root.mockResolvedValue(root);++ const result = await loadTraceTranscript({+ ...trace,+ transcriptSource: "root",+ });++ expect(mocks.observations).toHaveBeenCalledTimes(1);+ expect(mocks.assemble).not.toHaveBeenCalled();+ expect(result.cutoff).toBe(false);+ expect(result.transcript?.threads[0]?.currentTurn.messages).toMatchObject([+ { observationId: "root", parts: [{ type: "text", text: "Root question" }] },+ { observationId: "root", parts: [{ type: "text", text: "Root answer" }] },+ ]);+});++it.each(["missing", "empty"])(+ "does not use generations when root I/O is %s in root mode",+ async (rootState) => {+ mocks.observations.mockResolvedValue({+ observations:+ rootState === "missing"+ ? [+ {+ id: "generation",+ parentObservationId: "root",+ type: "GENERATION",+ },+ ]+ : [{ id: "root", parentObservationId: null, startTime: new Date(0) }],+ totalCount: 1,+ });+ mocks.root.mockResolvedValue({ input: null, output: null, metadata: {} });++ const result = await loadTraceTranscript({+ ...trace,+ transcriptSource: "root",+ });++ expect(result.transcript).toBeNull();+ expect(mocks.observations).toHaveBeenCalledTimes(1);+ expect(mocks.assemble).not.toHaveBeenCalled();+ },+);+it("leaves an empty transcript unchanged unless fallback is enabled", async () => {mocks.assemble.mockReturnValue(null);mocks.observations.mockResolvedValue({diff --git a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsxindex 3ec94c4367..336c533754 100644--- a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx+++ b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx@@ -150,6 +150,7 @@ type DropdownMenuSubmenu = {disabled?: { reason: string };id: string;title: string;+ internal?: boolean;icon?: IconComponent;items: DropdownMenuItemDefinition[];searchBehavior?: SearchBehavior;@@ -472,6 +473,13 @@ function DropdownMenuNode({<span className="min-w-0 flex-1 overflow-hidden text-left text-ellipsis whitespace-nowrap">{item.title}</span>+ {item.internal ? (+ <Badge+ text="Internal"+ color="yellow"+ size="sm"+ />+ ) : null}<span className="ml-2 flex"><DropdownIndicator direction="right" /></span>diff --git a/web/src/features/events/server/eventsRouter.ts b/web/src/features/events/server/eventsRouter.tsindex 4f5ecf388c..c6e732206b 100644--- a/web/src/features/events/server/eventsRouter.ts+++ b/web/src/features/events/server/eventsRouter.ts@@ -454,6 +454,7 @@ export const eventsRouter = createTRPCRouter({traceId: zodSchema.string(),timestamp: zodSchema.date().optional(),fallbackToRootIO: zodSchema.boolean().default(false),+ transcriptSource: zodSchema.enum(["generations", "root"]).optional(),}),).query(async ({ input, ctx }) => {@@ -502,6 +503,7 @@ export const eventsRouter = createTRPCRouter({traceId: input.traceId,timestamp,fallbackToRootIO: input.fallbackToRootIO,+ transcriptSource: input.transcriptSource,});},);diff --git a/web/src/features/events/server/loadTraceTranscript.ts b/web/src/features/events/server/loadTraceTranscript.tsindex 07646e677d..cb40e07947 100644--- a/web/src/features/events/server/loadTraceTranscript.ts+++ b/web/src/features/events/server/loadTraceTranscript.ts@@ -41,15 +41,18 @@ export async function loadTraceTranscript(trace: {timestamp: Date;/** Use root observation I/O when no generation/tool transcript exists. */fallbackToRootIO?: boolean;+ transcriptSource?: "generations" | "root";}): Promise<{ transcript: SessionTranscript | null; cutoff: boolean }> {const [structure, content] = await Promise.all([getObservationsForTraceFromEventsTable(trace),- getObservationsForTraceFromEventsTable({- ...trace,- selectIOAndMetadata: true,- types: ["GENERATION", "TOOL"],- limit: MAX_TRANSCRIPT_OBSERVATIONS,- }),+ trace.transcriptSource === "root"+ ? Promise.resolve({ observations: [], totalCount: 0 })+ : getObservationsForTraceFromEventsTable({+ ...trace,+ selectIOAndMetadata: true,+ types: ["GENERATION", "TOOL"],+ limit: MAX_TRANSCRIPT_OBSERVATIONS,+ }),]);const contentById = new Map(content.observations.map((o) => [o.id, o]));// Generations and tools past the content cap have no I/O to show; leaving@@ -90,8 +93,14 @@ export async function loadTraceTranscript(trace: {}return TRANSCRIPT_OBSERVATION_TYPES.has(o.type) ? [] : [o];});- let transcript = assembleTranscript(orderObservations(observations));- if (!transcript && trace.fallbackToRootIO) {+ let transcript =+ trace.transcriptSource === "root"+ ? null+ : assembleTranscript(orderObservations(observations));+ if (+ !transcript &&+ (trace.transcriptSource === "root" || trace.fallbackToRootIO)+ ) {const root = structure.observations.find((observation) =>observation.isRootObservation === true ||diff --git a/web/src/features/sessions/ModernSessionHeaderActionsController.clienttest.tsx b/web/src/features/sessions/ModernSessionHeaderActionsController.clienttest.tsxindex 188210592b..15f581947f 100644--- a/web/src/features/sessions/ModernSessionHeaderActionsController.clienttest.tsx+++ b/web/src/features/sessions/ModernSessionHeaderActionsController.clienttest.tsx@@ -5,13 +5,22 @@ import { vi } from "vitest";import { type DropdownMenuItemDefinition } from "@/src/components/design-system/DropdownMenu/DropdownMenu";import { ModernSessionHeaderActionsController } from "@/src/features/sessions/ModernSessionHeaderActionsController";-const { mockUseSession, copy, copyTextToClipboard } = vi.hoisted(() => ({+const {+ mockUseSession,+ mockInternalFeaturesEnabled,+ copy,+ copyTextToClipboard,+} = vi.hoisted(() => ({mockUseSession: vi.fn(),+ mockInternalFeaturesEnabled: vi.fn(),copy: vi.fn(),copyTextToClipboard: vi.fn(),}));vi.mock("next-auth/react", () => ({ useSession: mockUseSession }));+vi.mock("@/src/features/feature-flags", () => ({+ useInternalFeaturesEnabled: mockInternalFeaturesEnabled,+}));vi.mock("next/router", () => ({useRouter: () => ({ asPath: "/", push: vi.fn() }),@@ -49,7 +58,10 @@ function FlatMenu({ items }: { items: DropdownMenuItemDefinition[] }) {if (item.type === "submenu") {return (<div key={item.id}>- <div>{item.title}</div>+ <div>+ {item.title}+ {item.internal ? " Internal" : null}+ </div><FlatMenu items={item.items} /></div>);@@ -103,6 +115,7 @@ vi.mock("@/src/utils/clipboard", () => ({ copyTextToClipboard }));describe("ModernSessionHeaderActionsController", () => {beforeEach(() => {vi.clearAllMocks();+ mockInternalFeaturesEnabled.mockReturnValue(true);mockUseSession.mockReturnValue({ data: { user: { admin: false } } });});@@ -211,4 +224,41 @@ describe("ModernSessionHeaderActionsController", () => {expect(onShowInlineToolCallsChange).toHaveBeenCalledWith(true);});++ it("changes the transcript source from the actions display menu", () => {+ const onUseRootIOChange = vi.fn();+ render(+ <ModernSessionHeaderActionsController+ projectId="project-id"+ sessionId="session-id"+ isPublic={false}+ useRootIO={false}+ onUseRootIOChange={onUseRootIOChange}+ />,+ );++ expect(screen.getByText("Display Internal")).toBeInTheDocument();+ fireEvent.click(+ screen.getByRole("menuitemcheckbox", { name: "Use root input/output" }),+ );+ expect(onUseRootIOChange).toHaveBeenCalledWith(true);+ });++ it("hides the timeline Display submenu without internal access", () => {+ mockInternalFeaturesEnabled.mockReturnValue(false);+ render(+ <ModernSessionHeaderActionsController+ projectId="project-id"+ sessionId="session-id"+ isPublic={false}+ useRootIO={false}+ onUseRootIOChange={vi.fn()}+ />,+ );++ expect(screen.queryByText(/Display/)).not.toBeInTheDocument();+ expect(+ screen.queryByRole("menuitemcheckbox", { name: "Use root input/output" }),+ ).not.toBeInTheDocument();+ });});diff --git a/web/src/features/sessions/ModernSessionHeaderActionsController.tsx b/web/src/features/sessions/ModernSessionHeaderActionsController.tsxindex 8173eb8ec5..1e7f0c5456 100644--- a/web/src/features/sessions/ModernSessionHeaderActionsController.tsx+++ b/web/src/features/sessions/ModernSessionHeaderActionsController.tsx@@ -1,5 +1,6 @@import { CopyIcon, MoreVertical } from "@langfuse/icons";import { useSession } from "next-auth/react";+import { useInternalFeaturesEnabled } from "@/src/features/feature-flags";import { HeaderActionButton } from "@/src/components/HeaderActionButton";import { HeaderActionMenuRows } from "@/src/components/HeaderActionMenuRow";@@ -48,20 +49,30 @@ type DisplaySettingProps = {showCorrections?: boolean;showInlineToolCalls?: boolean;showSystemPrompt?: boolean;+ useRootIO?: boolean;onShowCorrectionsChange?: (isEnabled: boolean) => void;onShowInlineToolCallsChange?: (isEnabled: boolean) => void;onShowSystemPromptChange?: (isEnabled: boolean) => void;+ onUseRootIOChange?: (isEnabled: boolean) => void;};function buildDisplayItems({showCorrections,showInlineToolCalls,showSystemPrompt,+ useRootIO,onShowCorrectionsChange,onShowInlineToolCallsChange,onShowSystemPromptChange,+ onUseRootIOChange,}: DisplaySettingProps): DropdownMenuItemDefinition[] {const settings = [+ {+ id: "use-root-io",+ title: "Use root input/output",+ checked: useRootIO,+ onChange: onUseRootIOChange,+ },{id: "show-corrections",title: "Show corrections",@@ -123,7 +134,11 @@ export function ModernSessionHeaderActionsController({objectId: sessionId,isPublic,});- const displayItems = buildDisplayItems(displaySettings);+ const internalFeaturesEnabled = useInternalFeaturesEnabled();+ const displayItems = buildDisplayItems({+ ...displaySettings,+ useRootIO: internalFeaturesEnabled ? displaySettings.useRootIO : undefined,+ });const adminItems: DropdownMenuItemDefinition[] =session.data?.user?.admin === true@@ -166,6 +181,7 @@ export function ModernSessionHeaderActionsController({type: "submenu" as const,id: "display",title: "Display",+ internal: displaySettings.useRootIO !== undefined,items: displayItems,},]diff --git a/web/src/features/sessions/SessionPages.tsx b/web/src/features/sessions/SessionPages.tsxindex 639cfc01bb..27d4620786 100644--- a/web/src/features/sessions/SessionPages.tsx+++ b/web/src/features/sessions/SessionPages.tsx@@ -1323,6 +1323,10 @@ const LoadedSessionEventsPage: React.FC<{};}, [positionInTraceColumn, sessionEventsTableName]);const [urlFiltersQuery] = useQueryParam("filter", StringParam);+ const [transcriptSource, setTranscriptSource] = useQueryParam(+ "transcriptSource",+ StringParam,+ );const filtersQuery = React.useMemo(() =>urlFiltersQuery ??@@ -1928,7 +1932,13 @@ const LoadedSessionEventsPage: React.FC<{onShowSystemPromptChange:setShowSystemPromptForSession,}- : {})}+ : {+ useRootIO: transcriptSource === "root",+ onUseRootIOChange: (checked: boolean) =>+ setTranscriptSource(+ checked ? "root" : "generations",+ ),+ })}/>)}</>@@ -1943,6 +1953,13 @@ const LoadedSessionEventsPage: React.FC<{sessionId={sessionId}isPublic={session.public}layout="menu"+ {...(isModernSessionEnabled && isSessionTimelineEnabled+ ? {+ useRootIO: transcriptSource === "root",+ onUseRootIOChange: (checked: boolean) =>+ setTranscriptSource(checked ? "root" : "generations"),+ }+ : {})}/><CommentDrawerControllerprojectId={projectId}diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsxindex bb23e8708c..3b7bca1b10 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx@@ -1,4 +1,6 @@import { useRef, useState } from "react";+import { StringParam, useQueryParam } from "use-query-params";+import { useInternalFeaturesEnabled } from "@/src/features/feature-flags";import {ConnectedSessionConversationTimeline,@@ -34,6 +36,15 @@ export function ConnectedModernSessionBodyTimeline({sessionId,openPeek,}: ConnectedModernSessionBodyTimelineProps) {+ const [transcriptSourceParam] = useQueryParam(+ "transcriptSource",+ StringParam,+ );+ const internalFeaturesEnabled = useInternalFeaturesEnabled();+ const transcriptSource =+ internalFeaturesEnabled && transcriptSourceParam === "root"+ ? "root"+ : "generations";const traces =tracesState.type === "loaded" ? tracesState.traces : EMPTY_TRACES;const [timelineVisibleTraceIds, setTimelineVisibleTraceIds] = useState<@@ -99,6 +110,7 @@ export function ConnectedModernSessionBodyTimeline({projectId,traces: traces.map((trace, index) => ({ trace, turnNumber: index + 1 })),activeTraceIds: activeTranscriptTraceIds,+ transcriptSource,});const allTranscriptsLoaded =tracesState.type === "loaded" &&@@ -109,6 +121,7 @@ export function ConnectedModernSessionBodyTimeline({traceId: trace.id,timestamp: trace.timestamp,fallbackToRootIO: true,+ transcriptSource,}) !== undefined,);const searchQuery = allTranscriptsLoadeddiff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsxindex c996cbc5fe..765161380d 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx@@ -39,15 +39,22 @@ function setup(client = new QueryClient()) {return {client,...renderHook(- () =>+ ({+ transcriptSource,+ }: Pick<+ Parameters<typeof useSessionTraceTranscripts>[0],+ "transcriptSource"+ >) =>useSessionTraceTranscripts({projectId: "project",+ transcriptSource,traces,activeTraceIds: new Set(traces.slice(0, 6).map(({ trace }) => trace.id),),}),{+ initialProps: { transcriptSource: "generations" },wrapper: ({ children }: { children: ReactNode }) => (<QueryClientProvider client={client}>{children}</QueryClientProvider>),@@ -60,6 +67,34 @@ beforeEach(() => {query.mockReset();});+it("uses separate cached transcripts when switching sources", async () => {+ query.mockResolvedValue(output);+ const { rerender, result, unmount, client } = setup();+ await waitFor(() =>+ expect(result.current.get("trace-5")?.state).toBe("loaded"),+ );+ expect(query).toHaveBeenCalledTimes(6);++ rerender({ transcriptSource: "root" });+ await waitFor(() => expect(query).toHaveBeenCalledTimes(12));+ await waitFor(() =>+ expect(result.current.get("trace-5")?.state).toBe("loaded"),+ );+ expect(+ query.mock.calls+ .slice(6)+ .every(([input]) => input.transcriptSource === "root"),+ ).toBe(true);++ rerender({ transcriptSource: "generations" });+ await waitFor(() =>+ expect(result.current.get("trace-5")?.state).toBe("loaded"),+ );+ expect(query).toHaveBeenCalledTimes(12);+ unmount();+ client.clear();+});+it("bounds concurrent requests and releases slots after success and failure", async () => {const pending: Array<{resolve: (value: typeof output) => void;diff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts b/web/src/features/sessions/hooks/useSessionTraceTranscripts.tsindex 28aa426f38..3e326f6e40 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts@@ -2,7 +2,7 @@ import { useState } from "react";import { useQueries } from "@tanstack/react-query";import { getQueryKey } from "@trpc/react-query";import PQueue from "p-queue";-import { api, type RouterOutputs } from "@/src/utils/api";+import { api, type RouterInputs, type RouterOutputs } from "@/src/utils/api";import { type EventSessionTrace } from "@/src/features/sessions/sessionDetailPageTypes";export type SessionTraceTranscriptState =@@ -14,10 +14,14 @@ export function useSessionTraceTranscripts({projectId,traces,activeTraceIds,+ transcriptSource,}: {projectId: string;traces: readonly { trace: EventSessionTrace; turnNumber: number }[];activeTraceIds: ReadonlySet<string>;+ transcriptSource: NonNullable<+ RouterInputs["events"]["transcriptByTraceId"]["transcriptSource"]+ >;}): Map<string, SessionTraceTranscriptState> {const [queue] = useState(() => new PQueue({ concurrency: 4 }));const utils = api.useUtils();@@ -31,6 +35,7 @@ export function useSessionTraceTranscripts({traceId: trace.id,timestamp: trace.timestamp,fallbackToRootIO: true,+ transcriptSource,};return {queryKey: getQueryKey(api.events.transcriptByTraceId, input, "query"),