🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary, @typescript-eslint/no-use-before-define */
vi.mock("@langfuse/shared/src/server/llm/llmText", async () => {
const actual = await vi.importActual(
"@langfuse/shared/src/server/llm/llmText",
);
return {
...actual,
generateLLMText: vi.fn(),
};
});
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type { Session } from "next-auth";
import { EventType } from "@ag-ui/core";
import { randomUUID } from "crypto";
import { vi } from "vitest";
import waitForExpect from "wait-for-expect";
import { type Plan } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
createOrgProjectAndApiKey,
getScoreById,
} from "@langfuse/shared/src/server";
import { generateLLMText } from "@langfuse/shared/src/server/llm/llmText";
import { env } from "@/src/env.mjs";
import {
InAppAgentRunErrorCode,
InAppAgentRunStatus,
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
dropEmptyAssistantMessages,
dropUnpairedAssistantToolCalls,
type AgUiEvent,
type InAppAgentRunRequest,
IN_APP_AGENT_REDIRECT_TOOL_NAME,
} from "@langfuse/shared/in-app-agent";
import {
createInAppAgentConversationId,
createInAppAgentRunId,
} from "@/src/features/in-app-agent/ids";
import type { InAppAgentWatchFrame } from "@/src/features/in-app-agent/watchFrames";
import {
deserializeInAppAgentDisplayState,
projectInAppAgentMessagesForDisplay,
} from "@/src/features/in-app-agent/lib/display";
import { inAppAgentRouter } from "@/src/features/in-app-agent/server/router";
import {
ensureOwnedConversation,
getConversationEvents,
getConversationMessagesForReplay,
maybeInferAndPersistConversationTitle,
appendRunEvents,
flushPendingRunEvents,
shouldFlushPersistedEvent,
toPersistableAgentEvent,
} from "@langfuse/shared/in-app-agent/server/persistence";
import { finishClaimedRun } from "@langfuse/shared/in-app-agent/server/runLifecycle";
import { watchConversationFrames } from "@/src/features/in-app-agent/server/watch";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
vi.mock("@/src/server/auth", () => ({
getServerAuthSession: vi.fn(),
}));
const mockGenerateLLMText = vi.mocked(generateLLMText);
describe("in-app agent persistence", () => {
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
beforeEach(() => {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
});
afterEach(() => {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
mockGenerateLLMText.mockReset();
});
const createCaller = async (
userId = `user-${randomUUID()}`,
plan: Plan = "cloud:hobby",
) => {
const setup = await createOrgProjectAndApiKey();
await prisma.organization.update({
where: { id: setup.orgId },
data: { aiFeaturesEnabled: true },
});
await prisma.user.create({
data: {
id: userId,
email: `${userId}@example.com`,
},
});
const session: Session = {
expires: "1",
user: {
id: userId,
name: "Agent User",
canCreateOrganizations: true,
organizations: [
{
id: setup.orgId,
role: "OWNER",
plan,
cloudConfig: undefined,
name: "Test Organization",
metadata: {},
aiFeaturesEnabled: true,
aiTelemetryEnabled: true,
projects: [
{
id: setup.projectId,
role: "ADMIN",
name: "Test Project",
deletedAt: null,
retentionDays: null,
hasTraces: false,
metadata: {},
createdAt: new Date().toISOString(),
},
],
},
],
featureFlags: testFeatureFlags({ templateFlag: false }),
admin: false,
},
environment: {} as any,
};
const ctx = createInnerTRPCContext({ session, headers: {} });
return {
...setup,
userId,
caller: inAppAgentRouter.createCaller({ ...ctx, prisma }),
session,
};
};
const createConversation = (params: {
projectId: string;
userId: string;
conversationId?: string;
}) =>
ensureOwnedConversation({
prisma,
projectId: params.projectId,
conversationId: params.conversationId ?? createInAppAgentConversationId(),
userId: params.userId,
});
// Create only the claimed run record so persistence tests can control their
// exact event rows while exercising transitions through lifecycle APIs.
const createClaimedRunFixture = async (params: {
projectId: string;
conversationId: string;
userId: string;
runId?: string;
request?: InAppAgentRunRequest;
}) => {
const now = new Date();
return prisma.inAppAgentRun.create({
data: {
id: params.runId ?? createInAppAgentRunId(),
projectId: params.projectId,
conversationId: params.conversationId,
triggeredByUserId: params.userId,
model: "haiku",
mcpApiKeyId: "api-key-id-1",
status: InAppAgentRunStatus.RUNNING,
claimedAt: now,
heartbeatAt: now,
...(params.request ? { request: params.request } : {}),
},
});
};
const finishConversationRun = (params: {
projectId: string;
runId: string;
errorCode?: InAppAgentRunErrorCode;
errorMessage?: string;
}) =>
finishClaimedRun({
prisma,
...params,
status:
params.errorCode == null
? InAppAgentRunStatus.SUCCEEDED
: params.errorCode === InAppAgentRunErrorCode.CANCELLED
? InAppAgentRunStatus.CANCELLED
: InAppAgentRunStatus.FAILED,
});
it("finishes claimed runs once and preserves the first terminal error", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await expect(
finishClaimedRun({
prisma,
projectId,
runId: run.id,
status: InAppAgentRunStatus.FAILED,
errorCode: InAppAgentRunErrorCode.AGENT_ERROR,
errorMessage: "Original agent error",
}),
).resolves.toBe(true);
await expect(
finishClaimedRun({
prisma,
projectId,
runId: run.id,
status: InAppAgentRunStatus.CANCELLED,
errorCode: InAppAgentRunErrorCode.CANCELLED,
errorMessage: "Client aborted request",
}),
).resolves.toBe(false);
await expect(
prisma.inAppAgentRun.findUniqueOrThrow({
where: { id_projectId: { id: run.id, projectId } },
select: {
status: true,
finishedAt: true,
errorCode: true,
errorMessage: true,
},
}),
).resolves.toEqual({
status: InAppAgentRunStatus.FAILED,
finishedAt: expect.any(Date),
errorCode: InAppAgentRunErrorCode.AGENT_ERROR,
errorMessage: "Original agent error",
});
});
it("allows oss-plan users to list conversations", async () => {
const { caller, projectId } = await createCaller(
`user-${randomUUID()}`,
"oss",
);
await expect(caller.listConversations({ projectId })).resolves.toEqual({
conversations: [],
nextCursor: undefined,
});
});
const startCompactRun = async (params: {
projectId: string;
conversationId: string;
runId: string;
messageId: string;
content: string;
}) => {
const userMessage = {
id: params.messageId,
role: "user" as const,
content: params.content,
};
const events: AgUiEvent[] = [];
await appendRunEvents({
prisma,
projectId: params.projectId,
conversationId: params.conversationId,
runId: params.runId,
events: [
{
type: EventType.RUN_STARTED,
threadId: params.conversationId,
runId: params.runId,
input: {
threadId: params.conversationId,
runId: params.runId,
state: null,
messages: [userMessage],
tools: [],
context: [],
forwardedProps: {},
},
},
],
});
return events;
};
const processAndPersistEvent = async (params: {
projectId: string;
conversationId: string;
runId: string;
events: AgUiEvent[];
event: AgUiEvent;
}) => {
const persistedEvent = toPersistableAgentEvent(params.event);
if (!persistedEvent) {
return;
}
params.events.push(persistedEvent);
if (!shouldFlushPersistedEvent(persistedEvent)) {
return;
}
await flushPendingRunEvents({
prisma,
projectId: params.projectId,
conversationId: params.conversationId,
runId: params.runId,
pendingEvents: params.events,
});
};
const appendAssistantText = async (params: {
projectId: string;
conversationId: string;
runId: string;
events: AgUiEvent[];
messageId: string;
chunks: string[];
}) => {
await processAndPersistEvent({
...params,
event: {
type: EventType.TEXT_MESSAGE_START,
messageId: params.messageId,
role: "assistant",
},
});
for (const delta of params.chunks) {
await processAndPersistEvent({
...params,
event: {
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: params.messageId,
delta,
},
});
}
await processAndPersistEvent({
...params,
event: {
type: EventType.TEXT_MESSAGE_END,
messageId: params.messageId,
},
});
};
it("stores compacted events and restores multi-turn messages", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run1 = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await prisma.inAppAgentConversation.update({
where: { id_projectId: { id: conversation.id, projectId } },
data: { providerSessionId: "claude-session-secret" },
});
const events1 = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run1.id,
messageId: "user-message-1",
content: "Please inspect today's traces for outliers",
});
const sentinelCreatedAt = new Date("2020-01-02T03:04:05.000Z");
await prisma.inAppAgentEvent.updateMany({
where: {
projectId,
conversationId: conversation.id,
runId: run1.id,
},
data: { createdAt: sentinelCreatedAt },
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: run1.id,
events: events1,
messageId: "assistant-message-1",
chunks: ["I will inspect recent traces", " and look for outliers."],
});
await finishConversationRun({
runId: run1.id,
projectId,
});
const run2 = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events2 = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run2.id,
messageId: "user-message-2",
content: "Inspect the next trace",
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run2.id,
events: events2,
event: {
type: EventType.TEXT_MESSAGE_CHUNK,
messageId: "assistant-message-2",
delta: "Next trace",
},
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run2.id,
events: events2,
event: {
type: EventType.TEXT_MESSAGE_CHUNK,
messageId: "assistant-message-2",
delta: " inspected.",
},
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run2.id,
events: events2,
event: {
type: EventType.RUN_FINISHED,
threadId: conversation.id,
runId: run2.id,
},
});
const detail = await caller.getConversation({
projectId,
conversationId: conversation.id,
});
expect(detail.conversation).not.toHaveProperty("providerSessionId");
expect(detail.conversation).not.toHaveProperty("provider");
expect(detail.state).toEqual({
type: "existingConversation",
projectId,
conversationId: conversation.id,
});
expect(detail.messages).toEqual([
{
id: "user-message-1",
role: "user",
content: "Please inspect today's traces for outliers",
},
{
id: "assistant-message-1",
role: "assistant",
content: "I will inspect recent traces and look for outliers.",
runId: run1.id,
},
{
id: "user-message-2",
role: "user",
content: "Inspect the next trace",
},
{
id: "assistant-message-2",
role: "assistant",
content: "Next trace inspected.",
runId: run2.id,
},
]);
const events = await prisma.inAppAgentEvent.findMany({
where: { projectId, conversationId: conversation.id },
orderBy: { sequenceNumber: "asc" },
select: {
sequenceNumber: true,
type: true,
event: true,
createdAt: true,
},
});
expect(events.map((event) => event.sequenceNumber)).toEqual([
0, 1, 2, 3, 4, 5, 6,
]);
expect(events[0]?.createdAt).toEqual(sentinelCreatedAt);
expect(events.map((event) => event.type)).toEqual([
EventType.RUN_STARTED,
EventType.TEXT_MESSAGE_START,
EventType.TEXT_MESSAGE_CONTENT,
EventType.TEXT_MESSAGE_END,
EventType.RUN_STARTED,
EventType.TEXT_MESSAGE_CHUNK,
EventType.RUN_FINISHED,
]);
expect(events[0]?.event).toMatchObject({
type: EventType.RUN_STARTED,
input: {
messages: [
{
id: "user-message-1",
role: "user",
content: "Please inspect today's traces for outliers",
},
],
},
});
expect(events[2]?.event).toMatchObject({
type: EventType.TEXT_MESSAGE_CONTENT,
delta: "I will inspect recent traces and look for outliers.",
});
expect(events[4]?.event).toMatchObject({
type: EventType.RUN_STARTED,
input: {
messages: [
{
id: "user-message-2",
role: "user",
content: "Inspect the next trace",
},
],
},
});
expect(events[6]?.event).toMatchObject({
type: EventType.RUN_FINISHED,
});
expect(events[5]?.event).toMatchObject({
type: EventType.TEXT_MESSAGE_CHUNK,
messageId: "assistant-message-2",
role: "assistant",
delta: "Next trace inspected.",
});
const listedConversations = await caller.listConversations({ projectId });
expect(listedConversations.conversations.map((item) => item.id)).toContain(
conversation.id,
);
});
it("does not overwrite user-renamed conversation titles", async () => {
const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
await caller.renameConversation({
projectId,
conversationId: conversation.id,
title: "My custom title",
});
try {
(env as any).LANGFUSE_AI_SMALL_MODEL = "small-title-model";
await maybeInferAndPersistConversationTitle({
prisma,
projectId,
conversationId: conversation.id,
userId,
aiTelemetryEnabled: false,
});
await expect(
prisma.inAppAgentConversation.findUniqueOrThrow({
where: { id_projectId: { id: conversation.id, projectId } },
select: { title: true, renamedByUserAt: true },
}),
).resolves.toEqual({
title: "My custom title",
renamedByUserAt: expect.any(Date),
});
expect(mockGenerateLLMText).not.toHaveBeenCalled();
} finally {
(env as any).LANGFUSE_AI_SMALL_MODEL = originalBedrockSmallModel;
}
});
it("does not regenerate an already inferred conversation title", async () => {
const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
await prisma.inAppAgentConversation.update({
where: { id_projectId: { id: conversation.id, projectId } },
data: { title: "Cluster traces by tags" },
});
try {
(env as any).LANGFUSE_AI_SMALL_MODEL = "small-title-model";
await maybeInferAndPersistConversationTitle({
prisma,
projectId,
conversationId: conversation.id,
userId,
aiTelemetryEnabled: false,
});
await expect(
prisma.inAppAgentConversation.findUniqueOrThrow({
where: { id_projectId: { id: conversation.id, projectId } },
select: { title: true },
}),
).resolves.toEqual({ title: "Cluster traces by tags" });
expect(mockGenerateLLMText).not.toHaveBeenCalled();
} finally {
(env as any).LANGFUSE_AI_SMALL_MODEL = originalBedrockSmallModel;
}
});
it("keeps the default title when title generation fails", async () => {
const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const originalTitle = conversation.title;
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "failed-title-user",
content: "Inspect latency regressions",
});
try {
(env as any).LANGFUSE_AI_SMALL_MODEL = "small-title-model";
mockGenerateLLMText.mockRejectedValue(new Error("Bedrock failed"));
await expect(
maybeInferAndPersistConversationTitle({
prisma,
projectId,
conversationId: conversation.id,
userId,
aiTelemetryEnabled: false,
}),
).resolves.toBeUndefined();
await expect(
prisma.inAppAgentConversation.findUniqueOrThrow({
where: { id_projectId: { id: conversation.id, projectId } },
select: { title: true },
}),
).resolves.toEqual({ title: originalTitle });
} finally {
(env as any).LANGFUSE_AI_SMALL_MODEL = originalBedrockSmallModel;
}
});
it("scores feedback on the approval-chain root run and rejects run id mismatches", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const rootRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: rootRun.id,
messageId: "feedback-user",
content: "Answer me",
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: rootRun.id,
events,
messageId: "plain-assistant",
chunks: ["Answering directly"],
});
// The final answer comes from an approval continuation, but the trace it is
// recorded on belongs to the root run. The parent settles when it parks for
// approval, so only the continuation is active.
await finishConversationRun({ projectId, runId: rootRun.id });
const continuationRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
request: {
kind: "approvalDecision",
parentRunId: rootRun.id,
rootRunId: rootRun.id,
toolCallId: "tool-call-1",
approved: true,
context: [],
},
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: continuationRun.id,
events,
messageId: "feedback-assistant",
chunks: ["Here is an answer"],
});
await expect(
caller.submitFeedback({
projectId,
conversationId: conversation.id,
messageId: "feedback-assistant",
runId: createInAppAgentRunId(),
value: null,
comment: null,
}),
).rejects.toThrow(
"Feedback can only be submitted for persisted assistant messages",
);
const originalAiFeaturesProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;
try {
(env as any).LANGFUSE_AI_FEATURES_PROJECT_ID = projectId;
await expect(
caller.submitFeedback({
projectId,
conversationId: conversation.id,
messageId: "feedback-assistant",
runId: continuationRun.id,
value: "thumbs_up",
comment: null,
}),
).resolves.toEqual({ feedback: { value: "thumbs_up", comment: null } });
await expect(
caller.submitFeedback({
projectId,
conversationId: conversation.id,
messageId: "plain-assistant",
runId: rootRun.id,
value: "thumbs_down",
comment: null,
}),
).resolves.toEqual({ feedback: { value: "thumbs_down", comment: null } });
// Both the continuation's answer and the root run's own answer score onto
// the single trace the agent turn was recorded on.
await waitForExpect(async () => {
const [continuationScore, plainScore] = await Promise.all([
getScoreById({
projectId,
scoreId: `afbs_feedback-assistant_${userId}`,
}),
getScoreById({
projectId,
scoreId: `afbs_plain-assistant_${userId}`,
}),
]);
for (const score of [continuationScore, plainScore]) {
expect(score?.traceId).toBe(
getInAppAgentInstrumentationTraceId(rootRun.id),
);
expect(score?.observationId).toBe(
getInAppAgentInstrumentationObservationId(rootRun.id),
);
}
});
} finally {
(env as any).LANGFUSE_AI_FEATURES_PROJECT_ID =
originalAiFeaturesProjectId;
}
});
it("does not reduce partial assistant content before the end event", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "partial-user",
content: "Start a long answer",
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event: {
type: EventType.TEXT_MESSAGE_START,
messageId: "partial-assistant",
role: "assistant",
},
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event: {
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "partial-assistant",
delta: "Half-written",
},
});
await expect(
caller.getConversation({ projectId, conversationId: conversation.id }),
).resolves.toMatchObject({
messages: [
{
id: "partial-user",
role: "user",
content: "Start a long answer",
},
],
});
await expect(
prisma.inAppAgentEvent.count({
where: { projectId, conversationId: conversation.id, runId: run.id },
}),
).resolves.toBe(1);
});
it("hydrates visible reasoning without including it in model replay", async () => {
const { projectId, userId, caller } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "tool-user",
content: "Search traces",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "tool-assistant",
role: "assistant",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "tool-call-1",
toolCallName: "list_traces",
parentMessageId: "tool-assistant",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "tool-call-1",
delta: '{"limit":',
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "tool-call-1",
delta: "10}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "tool-call-1",
});
await process({
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "tool-assistant",
delta: "I searched traces.",
});
await process({
type: EventType.TEXT_MESSAGE_END,
messageId: "tool-assistant",
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "tool-result-1",
toolCallId: "tool-call-1",
content: "[]",
role: "tool",
});
await process({
type: EventType.REASONING_START,
});
await process({
type: EventType.REASONING_MESSAGE_START,
messageId: "reasoning-1",
role: "reasoning",
});
await process({
type: EventType.REASONING_MESSAGE_CONTENT,
messageId: "reasoning-1",
delta: "Checking ",
});
await process({
type: EventType.REASONING_MESSAGE_CONTENT,
messageId: "reasoning-1",
delta: "filters",
});
await process({
type: EventType.REASONING_ENCRYPTED_VALUE,
subtype: "message",
entityId: "reasoning-1",
encryptedValue: "encrypted-reasoning",
});
await process({
type: EventType.REASONING_MESSAGE_END,
messageId: "reasoning-1",
});
await process({
type: EventType.REASONING_END,
});
await process({
type: EventType.ACTIVITY_SNAPSHOT,
messageId: "activity-1",
activityType: "progress",
content: { status: "done" },
});
const expectedDisplayMessages = [
{
id: "tool-user",
role: "user" as const,
content: "Search traces",
},
{
id: "tool-assistant",
role: "assistant" as const,
content: "I searched traces.",
runId: run.id,
toolCalls: [
{
id: "tool-call-1",
type: "function" as const,
function: {
name: "list_traces",
arguments: '{"limit":10}',
},
},
],
},
{
id: "tool-result-1",
role: "tool" as const,
content: "[]",
toolCallId: "tool-call-1",
},
{
id: "reasoning-1",
role: "reasoning" as const,
content: "Checking filters",
},
{
id: "activity-1",
role: "activity" as const,
activityType: "progress",
content: { status: "done" },
},
];
const expectedReplayMessages = [
{
id: "tool-user",
role: "user" as const,
content: "Search traces",
},
{
id: "tool-assistant",
role: "assistant" as const,
content: "I searched traces.",
toolCalls: [
{
id: "tool-call-1",
type: "function" as const,
function: {
name: "list_traces",
arguments: '{"limit":10}',
},
},
],
},
{
id: "tool-result-1",
role: "tool" as const,
content: "[]",
toolCallId: "tool-call-1",
},
{
id: "activity-1",
role: "activity" as const,
activityType: "progress",
content: { status: "done" },
},
];
await expect(
caller.getConversation({ projectId, conversationId: conversation.id }),
).resolves.toMatchObject({
messages: expectedDisplayMessages,
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual(expectedReplayMessages);
const persistedEvents = await prisma.inAppAgentEvent.findMany({
where: { projectId, conversationId: conversation.id, runId: run.id },
orderBy: { sequenceNumber: "asc" },
select: { sequenceNumber: true, type: true, event: true },
});
expect(
persistedEvents
.filter((event) => event.type === EventType.REASONING_MESSAGE_CONTENT)
.map((event) => event.event),
).toEqual([
expect.objectContaining({
type: EventType.REASONING_MESSAGE_CONTENT,
delta: "Checking filters",
}),
]);
expect(JSON.stringify(persistedEvents)).not.toContain(
"encrypted-reasoning",
);
await prisma.inAppAgentEvent.create({
data: {
projectId,
conversationId: conversation.id,
runId: run.id,
sequenceNumber: (persistedEvents.at(-1)?.sequenceNumber ?? -1) + 1,
type: "FUTURE_REASONING_METADATA",
event: {
type: "FUTURE_REASONING_METADATA",
payload: "ignored",
},
},
});
await expect(
caller.getConversation({ projectId, conversationId: conversation.id }),
).resolves.toMatchObject({
messages: expectedDisplayMessages,
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual(expectedReplayMessages);
});
it("preserves reasoning order when hydrating an assistant message that continues afterward", async () => {
const { projectId, userId, caller } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await appendRunEvents({
prisma,
projectId,
conversationId: conversation.id,
runId: run.id,
events: [
{
type: EventType.RUN_STARTED,
threadId: conversation.id,
runId: run.id,
input: {
threadId: conversation.id,
runId: run.id,
state: null,
messages: [
{
id: "interleaved-user",
role: "user",
content: "Investigate this",
},
],
tools: [],
context: [],
forwardedProps: {},
},
},
{
type: EventType.TEXT_MESSAGE_START,
messageId: "interleaved-assistant",
role: "assistant",
},
{
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "interleaved-assistant",
delta: "Initial answer.",
},
{
type: EventType.REASONING_MESSAGE_START,
messageId: "interleaved-reasoning",
role: "reasoning",
},
{
type: EventType.REASONING_MESSAGE_CONTENT,
messageId: "interleaved-reasoning",
delta: "A later thought.",
},
{
type: EventType.REASONING_MESSAGE_END,
messageId: "interleaved-reasoning",
},
],
});
// Persist the continuation separately, as it would arrive after the
// snapshot prefix containing the interleaved reasoning message.
await appendRunEvents({
prisma,
projectId,
conversationId: conversation.id,
runId: run.id,
events: [
{
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "interleaved-assistant",
delta: " Final answer.",
},
{
type: EventType.TEXT_MESSAGE_END,
messageId: "interleaved-assistant",
},
],
});
// The wire carries canonical messages: the assistant message keeps its full
// content so a resumed run can append to it. Interleaved ordering travels
// separately, in the display state.
const snapshot = await caller.getConversation({
projectId,
conversationId: conversation.id,
});
expect(snapshot.messages).toMatchObject([
{
id: "interleaved-user",
role: "user",
content: "Investigate this",
},
{
id: "interleaved-assistant",
role: "assistant",
content: "Initial answer. Final answer.",
},
{
id: "interleaved-reasoning",
role: "reasoning",
content: "A later thought.",
},
]);
expect(
projectInAppAgentMessagesForDisplay(
snapshot.messages,
deserializeInAppAgentDisplayState(snapshot.displayState),
),
).toMatchObject([
{
id: "interleaved-user",
role: "user",
content: "Investigate this",
},
{
id: "interleaved-assistant",
role: "assistant",
content: "Initial answer.",
},
{
id: "interleaved-reasoning",
role: "reasoning",
content: "A later thought.",
},
{
id: "display-text-interleaved-assistant-1",
role: "assistant",
content: " Final answer.",
},
]);
});
it("stores only compact events and skips raw adapter payloads", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const eventsBuffer = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "safe-user",
content: "visible user text",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events: eventsBuffer,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-safe",
role: "assistant",
rawEvent: { token: "raw-text-start-secret" },
});
await process({
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "assistant-safe",
delta: "visible assistant text",
rawEvent: { token: "raw-text-content-secret" },
providerSessionId: "content-secret",
});
for (const event of [
{
type: EventType.CUSTOM,
name: "system:status",
value: { session_id: "provider-status-secret" },
},
{
type: EventType.STATE_DELTA,
delta: [
{
op: "replace",
path: "",
value: { providerSessionId: "state-delta-secret" },
},
],
},
{
type: EventType.REASONING_MESSAGE_CONTENT,
messageId: "reasoning-1",
delta: "reasoning-secret",
},
{
type: EventType.RAW,
event: { Authorization: "Basic raw-secret" },
},
]) {
await process(event);
}
const events = await prisma.inAppAgentEvent.findMany({
where: { projectId, conversationId: conversation.id, runId: run.id },
orderBy: { sequenceNumber: "asc" },
select: { type: true, event: true },
});
expect(events.map((event) => event.type)).toEqual([EventType.RUN_STARTED]);
expect(events[0]?.event).toMatchObject({
type: EventType.RUN_STARTED,
input: {
messages: [
{
id: "safe-user",
role: "user",
content: "visible user text",
},
],
},
});
expect(JSON.stringify(events)).not.toContain("secret");
});
it("does not persist adapter message snapshots", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "snapshot-user",
content: "Keep this",
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event: {
type: EventType.MESSAGES_SNAPSHOT,
messages: [
{
id: "snapshot-only",
role: "assistant",
content: "Restore me from the snapshot",
},
],
},
});
await expect(
caller.getConversation({ projectId, conversationId: conversation.id }),
).resolves.toMatchObject({
messages: [
{
id: "snapshot-user",
role: "user",
content: "Keep this",
},
],
});
await expect(
prisma.inAppAgentEvent.count({
where: { projectId, conversationId: conversation.id, runId: run.id },
}),
).resolves.toBe(1);
});
it("drops trailing user-only turns before replay", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const completedRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const completedEvents = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
messageId: "user-1",
content: "first",
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
events: completedEvents,
event: {
type: EventType.TEXT_MESSAGE_CHUNK,
messageId: "assistant-1",
delta: "done",
},
});
await processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
events: completedEvents,
event: {
type: EventType.RUN_FINISHED,
threadId: conversation.id,
runId: completedRun.id,
},
});
await finishConversationRun({
runId: completedRun.id,
projectId,
});
const abandonedRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await startCompactRun({
projectId,
conversationId: conversation.id,
runId: abandonedRun.id,
messageId: "orphan-user-1",
content: "failed",
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual([
{ id: "user-1", role: "user", content: "first" },
{ id: "assistant-1", role: "assistant", content: "done" },
{ id: "orphan-user-1", role: "user", content: "failed" },
]);
});
it("drops assistant tool calls that have no matching tool result", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "user-1",
content: "search",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-1",
role: "assistant",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "paired-tool-call",
toolCallName: "list_traces",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "paired-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "paired-tool-call",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "orphan-tool-call",
toolCallName: "get_trace",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "orphan-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "orphan-tool-call",
});
await process({
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "assistant-1",
delta: "calling tools",
});
await process({
type: EventType.TEXT_MESSAGE_END,
messageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "tool-result-1",
toolCallId: "paired-tool-call",
content: "[]",
role: "tool",
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual([
{ id: "user-1", role: "user", content: "search" },
{
id: "assistant-1",
role: "assistant",
content: "calling tools",
toolCalls: [
{
id: "paired-tool-call",
type: "function",
function: { name: "list_traces", arguments: "{}" },
},
],
},
{
id: "tool-result-1",
role: "tool",
content: "[]",
toolCallId: "paired-tool-call",
},
]);
});
it("redacts silent MCP output from replayed conversation history", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "user-1",
content: "search observations silently",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "tool-call-1",
toolCallName: "langfuse_listObservations",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "tool-call-1",
delta: JSON.stringify({ silent: true, traceId: "trace-1" }),
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "tool-call-1",
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "tool-result-1",
toolCallId: "tool-call-1",
content: JSON.stringify({
type: "silent-mcp-output",
output: { data: [{ id: "observation-1" }] },
}),
role: "tool",
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toContainEqual({
id: "tool-result-1",
role: "tool",
content: "Output saved to /workspace/tool_calls",
toolCallId: "tool-call-1",
});
const persistedEvents = await getConversationEvents({
prisma,
projectId,
conversationId: conversation.id,
});
expect(persistedEvents).toContainEqual(
expect.objectContaining({
event: expect.objectContaining({
type: EventType.TOOL_CALL_ARGS,
delta: JSON.stringify({ silent: true, traceId: "trace-1" }),
}),
}),
);
});
it("drops assistant tool calls without results from loaded conversation history", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "user-1",
content: "search",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-1",
role: "assistant",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "paired-tool-call",
toolCallName: "list_traces",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "paired-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "paired-tool-call",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "unapproved-tool-call",
toolCallName: "get_trace",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "unapproved-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "unapproved-tool-call",
});
await process({
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "assistant-1",
delta: "calling tools",
});
await process({
type: EventType.TEXT_MESSAGE_END,
messageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "tool-result-1",
toolCallId: "paired-tool-call",
content: "[]",
role: "tool",
});
const detail = await caller.getConversation({
projectId,
conversationId: conversation.id,
});
// The snapshot stays canonical and keeps the unpaired call: a run resuming
// from here still needs it present for its result to attach to, and a
// pending approval renders against it.
expect(
detail.messages.flatMap((message) =>
message.role === "assistant" ? (message.toolCalls ?? []) : [],
),
).toMatchObject([
{ id: "paired-tool-call" },
{ id: "unapproved-tool-call" },
]);
// Settled transcripts prune it at render time instead.
expect(
dropEmptyAssistantMessages(
dropUnpairedAssistantToolCalls(detail.messages),
),
).toEqual([
{ id: "user-1", role: "user", content: "search" },
{
id: "assistant-1",
role: "assistant",
content: "calling tools",
runId: run.id,
toolCalls: [
{
id: "paired-tool-call",
type: "function",
function: { name: "list_traces", arguments: "{}" },
},
],
},
{
id: "tool-result-1",
role: "tool",
content: "[]",
toolCallId: "paired-tool-call",
},
]);
});
it("flushes sibling tools while retaining only successful redirect actions for display", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "user-1",
content: "open a trace",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-1",
role: "assistant",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "redirect-tool-call",
toolCallName: IN_APP_AGENT_REDIRECT_TOOL_NAME,
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "redirect-tool-call",
delta: '{"destination":"trace"}',
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "redirect-tool-call",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "sibling-tool-call",
toolCallName: "list_traces",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "sibling-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "sibling-tool-call",
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "sibling-tool-result",
toolCallId: "sibling-tool-call",
content: "[]",
role: "tool",
});
const expectedReplayMessages = [
{ id: "user-1", role: "user" as const, content: "open a trace" },
{
id: "assistant-1",
role: "assistant" as const,
toolCalls: [
{
id: "sibling-tool-call",
type: "function" as const,
function: { name: "list_traces", arguments: "{}" },
},
],
},
{
id: "sibling-tool-result",
role: "tool" as const,
content: "[]",
toolCallId: "sibling-tool-call",
},
];
// A completed sibling tool unit must reach the Postgres-backed replay
// surface without waiting for an unrelated redirect result.
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual(expectedReplayMessages);
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "tool-result-1",
toolCallId: "redirect-tool-call",
content: "Tool validation failed: trace params are required",
role: "tool",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "successful-redirect-tool-call",
toolCallName: IN_APP_AGENT_REDIRECT_TOOL_NAME,
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "successful-redirect-tool-call",
delta: '{"destination":"trace"}',
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "successful-redirect-tool-call",
});
const redirectActionContent = JSON.stringify({
type: "redirectAction",
label: "Open trace",
href: `/project/${projectId}/traces/trace-1`,
});
await process({
type: EventType.TOOL_CALL_RESULT,
messageId: "successful-redirect-result",
toolCallId: "successful-redirect-tool-call",
content: redirectActionContent,
role: "tool",
});
// Successful redirect actions must survive refresh as render-only results,
// while failed redirects and their evolving call arguments stay absent.
await expect(
caller.getConversation({
projectId,
conversationId: conversation.id,
}),
).resolves.toMatchObject({
messages: [
{ id: "user-1", role: "user", content: "open a trace" },
{
id: "assistant-1",
role: "assistant",
runId: run.id,
toolCalls: [
{
id: "sibling-tool-call",
type: "function",
function: { name: "list_traces", arguments: "{}" },
},
],
},
{
id: "sibling-tool-result",
role: "tool",
content: "[]",
toolCallId: "sibling-tool-call",
},
{
id: "successful-redirect-result",
role: "tool",
content: redirectActionContent,
toolCallId: "successful-redirect-tool-call",
},
],
});
// Render-only redirect actions must never enter the next model request.
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual(expectedReplayMessages);
});
it("drops empty assistant messages after removing orphan tool calls before replay", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run.id,
messageId: "user-1",
content: "search",
});
const process = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: run.id,
events,
event,
});
await process({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-1",
role: "assistant",
});
await process({
type: EventType.TOOL_CALL_START,
toolCallId: "orphan-tool-call",
toolCallName: "get_trace",
parentMessageId: "assistant-1",
});
await process({
type: EventType.TOOL_CALL_ARGS,
toolCallId: "orphan-tool-call",
delta: "{}",
});
await process({
type: EventType.TOOL_CALL_END,
toolCallId: "orphan-tool-call",
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual([{ id: "user-1", role: "user", content: "search" }]);
});
it("keeps user messages before empty assistant messages removed from replay", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const orphanToolRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const orphanToolEvents = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: orphanToolRun.id,
messageId: "user-1",
content: "search",
});
const processOrphanToolEvent = (event: AgUiEvent) =>
processAndPersistEvent({
projectId,
conversationId: conversation.id,
runId: orphanToolRun.id,
events: orphanToolEvents,
event,
});
await processOrphanToolEvent({
type: EventType.TEXT_MESSAGE_START,
messageId: "assistant-1",
role: "assistant",
});
await processOrphanToolEvent({
type: EventType.TOOL_CALL_START,
toolCallId: "orphan-tool-call",
toolCallName: "get_trace",
parentMessageId: "assistant-1",
});
await processOrphanToolEvent({
type: EventType.TOOL_CALL_END,
toolCallId: "orphan-tool-call",
});
await finishConversationRun({
runId: orphanToolRun.id,
projectId,
errorCode: InAppAgentRunErrorCode.CANCELLED,
errorMessage: "Aborted before tool result",
});
const completedRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const completedEvents = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
messageId: "user-2",
content: "try again",
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
events: completedEvents,
messageId: "assistant-2",
chunks: ["done"],
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual([
{ id: "user-1", role: "user", content: "search" },
{ id: "user-2", role: "user", content: "try again" },
{ id: "assistant-2", role: "assistant", content: "done" },
]);
});
it("keeps interior user-only failed turns before replay", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const failedRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await startCompactRun({
projectId,
conversationId: conversation.id,
runId: failedRun.id,
messageId: "user-1",
content: "failed before output",
});
await finishConversationRun({
runId: failedRun.id,
projectId,
errorCode: InAppAgentRunErrorCode.AGENT_ERROR,
errorMessage: "Failed before output",
});
const completedRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const completedEvents = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
messageId: "user-2",
content: "try again",
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: completedRun.id,
events: completedEvents,
messageId: "assistant-2",
chunks: ["done"],
});
await expect(
getConversationMessagesForReplay({
prisma,
projectId,
conversationId: conversation.id,
}),
).resolves.toEqual([
{ id: "user-1", role: "user", content: "failed before output" },
{ id: "user-2", role: "user", content: "try again" },
{ id: "assistant-2", role: "assistant", content: "done" },
]);
});
it("does not expose another user's conversation in the same project", async () => {
const owner = await createCaller();
const otherUserId = `user-${randomUUID()}`;
await prisma.user.create({
data: {
id: otherUserId,
email: `${otherUserId}@example.com`,
},
});
const otherSession: Session = {
...owner.session,
user: {
...owner.session.user!,
id: otherUserId,
},
};
const otherCtx = createInnerTRPCContext({
session: otherSession,
headers: {},
});
const otherCaller = inAppAgentRouter.createCaller({ ...otherCtx, prisma });
const conversation = await createConversation({
projectId: owner.projectId,
userId: owner.userId,
});
await expect(
otherCaller.getConversation({
projectId: owner.projectId,
conversationId: conversation.id,
}),
).rejects.toThrow("Agent conversation not found");
const visibleConversations = await otherCaller.listConversations({
projectId: owner.projectId,
});
expect(visibleConversations.conversations).toEqual([]);
expect(visibleConversations.nextCursor).toBeUndefined();
});
it("allows conversation and run ids to repeat across projects", async () => {
const owner = await createCaller();
const other = await createCaller();
const conversation = await createConversation({
projectId: owner.projectId,
userId: owner.userId,
});
const otherConversation = await ensureOwnedConversation({
prisma,
projectId: other.projectId,
conversationId: conversation.id,
userId: other.userId,
});
expect(otherConversation).toMatchObject({
id: conversation.id,
projectId: other.projectId,
createdByUserId: other.userId,
});
const runId = createInAppAgentRunId();
await createClaimedRunFixture({
projectId: owner.projectId,
conversationId: conversation.id,
userId: owner.userId,
runId,
});
await expect(
createClaimedRunFixture({
projectId: other.projectId,
conversationId: otherConversation.id,
userId: other.userId,
runId,
}),
).resolves.toMatchObject({
id: runId,
projectId: other.projectId,
});
});
it("ignores event flushes from already-finished runs", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const run1 = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
const events1 = await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run1.id,
messageId: "user-message-1",
content: "First",
});
await finishConversationRun({ runId: run1.id, projectId });
const run2 = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await startCompactRun({
projectId,
conversationId: conversation.id,
runId: run2.id,
messageId: "user-message-2",
content: "Second",
});
await appendAssistantText({
projectId,
conversationId: conversation.id,
runId: run1.id,
events: events1,
messageId: "assistant-message-1",
chunks: ["Late output"],
});
const events = await prisma.inAppAgentEvent.findMany({
where: { projectId, conversationId: conversation.id },
orderBy: { sequenceNumber: "asc" },
select: { runId: true, type: true },
});
expect(events).toEqual([
{ runId: run1.id, type: EventType.RUN_STARTED },
{ runId: run2.id, type: EventType.RUN_STARTED },
]);
});
it("tails only post-cursor events, across runs, from one conversation-rooted read", async () => {
// The watch poll is a single joined read of the conversation's latest run
// and its post-cursor events (LFE-14629). Its unit suite fakes that read
// and reimplements the cursor filter in JS, so this is the only coverage
// that the nested where/orderBy/take actually compose against Postgres.
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const firstRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
await appendRunEvents({
prisma,
projectId,
conversationId: conversation.id,
runId: firstRun.id,
events: [
{
type: EventType.RUN_STARTED,
threadId: conversation.id,
runId: firstRun.id,
input: {
threadId: conversation.id,
runId: firstRun.id,
state: null,
messages: [{ id: "tail-user", role: "user", content: "Look" }],
tools: [],
context: [],
forwardedProps: {},
},
},
{
type: EventType.TEXT_MESSAGE_START,
messageId: "tail-first",
role: "assistant",
},
{
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "tail-first",
delta: "Parked answer.",
},
{ type: EventType.TEXT_MESSAGE_END, messageId: "tail-first" },
],
finish: { status: InAppAgentRunStatus.SUCCEEDED },
});
const secondRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
// A continuation's first persisted row is a plain event, not RUN_STARTED,
// which is what forces the synthetic open in the tail below.
await appendRunEvents({
prisma,
projectId,
conversationId: conversation.id,
runId: secondRun.id,
events: [
{
type: EventType.TEXT_MESSAGE_START,
messageId: "tail-second",
role: "assistant",
},
{
type: EventType.TEXT_MESSAGE_CONTENT,
messageId: "tail-second",
delta: "Continued answer.",
},
{
type: EventType.RUN_FINISHED,
threadId: conversation.id,
runId: secondRun.id,
},
],
finish: { status: InAppAgentRunStatus.SUCCEEDED },
});
// Sequence 0 is the first run's RUN_STARTED and 1 its assistant
// TEXT_MESSAGE_START, so a cursor of 1 starts the tail mid-run.
const frames: InAppAgentWatchFrame[] = [];
for await (const frame of watchConversationFrames({
prisma,
projectId,
conversationId: conversation.id,
cursor: 1,
now: () => 0,
sleep: async () => undefined,
})) {
if (frame !== null) {
frames.push(frame);
}
}
const relayed = frames.flatMap((frame) =>
frame.type === "event" ? [frame.event] : [],
);
// A dropped cursor predicate would relay sequences 0 and 1 too, adding the
// first run's persisted RUN_STARTED and a second TEXT_MESSAGE_START.
expect(relayed.map((event) => event.type)).toEqual([
EventType.RUN_STARTED,
EventType.TEXT_MESSAGE_CONTENT,
EventType.TEXT_MESSAGE_END,
EventType.RUN_FINISHED,
EventType.RUN_STARTED,
EventType.TEXT_MESSAGE_START,
EventType.TEXT_MESSAGE_CONTENT,
EventType.RUN_FINISHED,
]);
// The tail spans runs: the events relation is conversation-scoped, so a
// window can hold a parked parent's rows below its continuation's.
expect(
relayed.flatMap((event) => ("runId" in event ? [event.runId] : [])),
).toEqual([firstRun.id, firstRun.id, secondRun.id, secondRun.id]);
expect(frames.filter((frame) => frame.type === "status")).toEqual([
{
type: "status",
runId: secondRun.id,
status: InAppAgentRunStatus.SUCCEEDED,
errorCode: null,
cancelRequested: false,
},
]);
expect(frames.at(-1)).toEqual({ type: "done" });
});
it("enforces the single-active-run backstop index at the database level", async () => {
const { projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
});
// The partial unique index is the database-level invariant against two
// unfinished runs in one conversation. Prisma resolves the raw-SQL index
// to its column names in the conflict metadata.
await expect(
prisma.inAppAgentRun.create({
data: {
id: createInAppAgentRunId(),
projectId,
conversationId: conversation.id,
status: InAppAgentRunStatus.RUNNING,
},
}),
).rejects.toMatchObject({
code: "P2002",
meta: { target: ["project_id", "conversation_id"] },
});
});
it("paginates conversation list with a stable cursor", async () => {
const { caller, projectId, userId } = await createCaller();
const idPrefix = `pagination-${randomUUID()}`;
const latest = new Date("2026-05-20T10:00:00.000Z");
const middle = new Date("2026-05-20T09:00:00.000Z");
const oldest = new Date("2026-05-20T08:00:00.000Z");
await prisma.inAppAgentConversation.createMany({
data: [
{
id: `${idPrefix}-a`,
projectId,
createdByUserId: userId,
createdAt: latest,
updatedAt: latest,
},
{
id: `${idPrefix}-b`,
projectId,
createdByUserId: userId,
createdAt: latest,
updatedAt: latest,
},
{
id: `${idPrefix}-c`,
projectId,
createdByUserId: userId,
createdAt: middle,
updatedAt: middle,
},
{
id: `${idPrefix}-d`,
projectId,
createdByUserId: userId,
createdAt: oldest,
updatedAt: oldest,
},
],
});
const firstPage = await caller.listConversations({ projectId, limit: 2 });
expect(firstPage.conversations.map((item) => item.id)).toEqual([
`${idPrefix}-b`,
`${idPrefix}-a`,
]);
expect(firstPage.nextCursor).toEqual({
updatedAt: latest,
id: `${idPrefix}-a`,
});
const secondPage = await caller.listConversations({
projectId,
limit: 2,
cursor: firstPage.nextCursor,
});
expect(secondPage.conversations.map((item) => item.id)).toEqual([
`${idPrefix}-c`,
`${idPrefix}-d`,
]);
expect(secondPage.nextCursor).toBeUndefined();
});
});