🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod";
import {
ForbiddenError,
InvalidRequestError,
resolveInAppAgentRootRunId,
ScoreDataTypeEnum,
ScoreSourceEnum,
TEXT_SCORE_MAX_LENGTH,
} from "@langfuse/shared";
import {
convertDateToClickhouseDateTime,
upsertScore,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import {
AgUiContextSchema,
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
IN_APP_AGENT_PRODUCT_ENVIRONMENT,
InAppAgentToolInputsSchema,
} from "@langfuse/shared/in-app-agent";
import { InAppAgentMessageFeedbackValueSchema } from "../schema";
import {
assertInAppAgentAvailable,
assertInAppAgentModelConfigured,
} from "@/src/features/in-app-agent/server/availability";
import {
createTRPCRouter,
protectedProjectProcedure,
protectedProjectProcedureWithoutTracing,
} from "@/src/server/api/trpc";
import {
getConversationMessages,
getOwnedConversationOrThrow,
serializeConversation,
} from "@langfuse/shared/in-app-agent/server/persistence";
import {
assertInAppAgentRateLimit,
getInAppAgentApiAccessScope,
} from "@/src/features/in-app-agent/server/rateLimit";
import { assertInAppAgentRunCapacity } from "@/src/features/in-app-agent/server/runCapacity";
import { hasInternalAccess } from "@/src/features/feature-flags/utils";
import {
cancelBackgroundRun,
decideBackgroundApproval,
deleteBackgroundConversation,
getBackgroundConversationSnapshot,
serializeConversationLatestRun,
startBackgroundRun,
} from "@/src/features/in-app-agent/server/backgroundRunService";
const CONVERSATION_LIST_LIMIT = 50;
const MAX_IN_APP_AGENT_MESSAGE_LENGTH = 32_000;
const ConversationListCursorSchema = z.object({
updatedAt: z.date(),
id: z.string(),
});
const ConversationIdInput = z.object({
projectId: z.string(),
conversationId: z.string(),
});
const RenameConversationInput = ConversationIdInput.extend({
title: z.string().trim().min(1).max(80),
});
const StartRunInput = ConversationIdInput.extend({
message: z.string().trim().min(1).max(MAX_IN_APP_AGENT_MESSAGE_LENGTH),
/**
* The AG-UI context for the current page, quick action, and entry point;
* resolved and sanitized server-side, then stored for the worker to replay.
*/
context: z.array(AgUiContextSchema).default([]),
});
const CancelRunInput = ConversationIdInput.extend({
runId: z.string(),
});
const DecideToolApprovalInput = ConversationIdInput.extend({
runId: z.string(),
toolCallId: z.string(),
approved: z.boolean(),
// "conversation" also approves this call; it never means approve-without-run.
approvalScope: z.enum(["once", "conversation"]).default("once"),
/** Validated against the persisted tool call in the service. */
toolInputs: InAppAgentToolInputsSchema.optional(),
})
.refine((input) => input.approved || input.approvalScope === "once", {
message: "A rejection cannot grant a tool",
path: ["approvalScope"],
})
.refine((input) => input.approved || input.toolInputs === undefined, {
message: "A rejection cannot carry tool inputs",
path: ["toolInputs"],
});
const SubmitFeedbackInput = ConversationIdInput.extend({
messageId: z.string(),
runId: z.string(),
value: InAppAgentMessageFeedbackValueSchema.nullable(),
comment: z.string().trim().max(TEXT_SCORE_MAX_LENGTH).nullable().optional(),
});
const IN_APP_AGENT_FEEDBACK_SCORE_NAME = "in_app_agent_feedback";
const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = IN_APP_AGENT_PRODUCT_ENVIRONMENT;
export const inAppAgentRouter = createTRPCRouter({
// Each row carries its newest run summary for the activity poll and badges.
// Activity clients request the newest page only (see
// IN_APP_AGENT_ACTIVITY_LIST_LIMIT); quieter conversations outside that
// window are an accepted attention gap for now.
listConversations: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
cursor: ConversationListCursorSchema.optional(),
limit: z.number().int().min(1).max(CONVERSATION_LIST_LIMIT).default(50),
}),
)
.query(async ({ ctx, input }) => {
await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
const conversations = await ctx.prisma.inAppAgentConversation.findMany({
where: {
projectId: input.projectId,
createdByUserId: ctx.session.user.id,
deletedAt: null,
...(input.cursor
? {
OR: [
{ updatedAt: { lt: input.cursor.updatedAt } },
{
updatedAt: input.cursor.updatedAt,
id: { lt: input.cursor.id },
},
],
}
: {}),
},
orderBy: [{ updatedAt: "desc" }, { id: "desc" }],
take: input.limit + 1,
relationLoadStrategy: "join",
select: {
id: true,
title: true,
createdAt: true,
updatedAt: true,
runs: {
orderBy: { createdAt: "desc" },
take: 1,
select: {
id: true,
status: true,
errorCode: true,
cancelRequestedAt: true,
createdAt: true,
claimedAt: true,
heartbeatAt: true,
finishedAt: true,
},
},
},
});
const page = conversations.slice(0, input.limit);
const lastConversation = page.at(-1);
return {
conversations: page.map((conversation) => ({
...serializeConversation(conversation),
latestRun: serializeConversationLatestRun(conversation.runs[0]),
})),
nextCursor:
conversations.length > input.limit && lastConversation
? {
updatedAt: lastConversation.updatedAt,
id: lastConversation.id,
}
: undefined,
};
}),
getConversation: protectedProjectProcedureWithoutTracing
.input(ConversationIdInput)
.query(async ({ ctx, input }) => {
await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
return getBackgroundConversationSnapshot({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
userId: ctx.session.user.id,
});
}),
/**
* Submit a turn for background execution.
*
* Validates the request, then commits the run as QUEUED with its user message
* already appended — the events table is
* the render source from the instant of submit, so there is no optimistic UI
* state to survive a refresh. The BullMQ enqueue happens after commit; a
* failure there marks the run FAILED immediately rather than leaving a
* committed run nobody will execute.
*/
startRun: protectedProjectProcedureWithoutTracing
.input(StartRunInput)
.mutation(async ({ ctx, input }) => {
const projectAvailability = await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
const modelConfig = assertInAppAgentModelConfigured();
const rateLimitScope = getInAppAgentApiAccessScope(
ctx.session.user,
input.projectId,
projectAvailability,
);
await assertInAppAgentRateLimit(rateLimitScope, "in-app-agent-run");
// The concurrency ceiling is enforced inside `startBackgroundRun`, which
// owns the conversation lookup it has to reconcile against first.
return startBackgroundRun({
prisma: ctx.prisma,
projectId: input.projectId,
orgId: rateLimitScope.orgId,
plan: rateLimitScope.plan,
conversationId: input.conversationId,
userId: ctx.session.user.id,
message: input.message,
context: input.context,
isV4Enabled: ctx.session.user.v4BetaEnabled === true,
model: modelConfig.modelId,
aiTelemetryEnabled: projectAvailability.aiTelemetryEnabled,
});
}),
cancelRun: protectedProjectProcedureWithoutTracing
.input(CancelRunInput)
.mutation(async ({ ctx, input }) => {
await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
return cancelBackgroundRun({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
runId: input.runId,
userId: ctx.session.user.id,
});
}),
/**
* Decide a pending tool approval.
*
* The client sends only IDs and a boolean. The tool name and arguments are
* read server-side from the persisted interrupt event, so there is nothing
* to tamper with on the way back and no fingerprint to keep in sync.
*/
decideToolApproval: protectedProjectProcedureWithoutTracing
.input(DecideToolApprovalInput)
.mutation(async ({ ctx, input }) => {
const projectAvailability = await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
const modelConfig = assertInAppAgentModelConfigured();
const rateLimitScope = getInAppAgentApiAccessScope(
ctx.session.user,
input.projectId,
projectAvailability,
);
await assertInAppAgentRateLimit(rateLimitScope, "in-app-agent-run");
// A continuation is a fresh run that occupies a worker slot, and parked
// approvals accumulate without holding one, so this path needs the
// ceiling for the same reason it already needs the daily bucket.
await assertInAppAgentRunCapacity({
prisma: ctx.prisma,
orgId: rateLimitScope.orgId,
plan: rateLimitScope.plan,
userId: ctx.session.user.id,
});
return decideBackgroundApproval({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
runId: input.runId,
toolCallId: input.toolCallId,
approved: input.approved,
approvalScope: input.approvalScope,
toolInputs: input.toolInputs,
userId: ctx.session.user.id,
model: modelConfig.modelId,
});
}),
deleteConversation: protectedProjectProcedureWithoutTracing
.input(ConversationIdInput)
.mutation(async ({ ctx, input }) => {
await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
return deleteBackgroundConversation({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
userId: ctx.session.user.id,
});
}),
renameConversation: protectedProjectProcedureWithoutTracing
.input(RenameConversationInput)
.mutation(async ({ ctx, input }) => {
await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
await getOwnedConversationOrThrow({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
userId: ctx.session.user.id,
});
const conversation = await ctx.prisma.inAppAgentConversation.update({
where: {
id_projectId: {
id: input.conversationId,
projectId: input.projectId,
},
createdByUserId: ctx.session.user.id,
deletedAt: null,
},
data: {
title: input.title,
renamedByUserAt: new Date(),
},
});
return { conversation: serializeConversation(conversation) };
}),
submitFeedback: protectedProjectProcedureWithoutTracing
.input(SubmitFeedbackInput)
.mutation(async ({ ctx, input }) => {
const projectAvailability = await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
await getOwnedConversationOrThrow({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
userId: ctx.session.user.id,
});
const messages = await getConversationMessages({
prisma: ctx.prisma,
projectId: input.projectId,
conversationId: input.conversationId,
});
const targetMessage = messages.find(
(message) => message.id === input.messageId,
);
if (
targetMessage?.role !== "assistant" ||
typeof targetMessage.content !== "string" ||
targetMessage.content.length === 0
) {
throw new InvalidRequestError(
"Feedback can only be submitted for assistant text messages",
);
}
if (targetMessage.runId !== input.runId) {
throw new InvalidRequestError(
"Feedback can only be submitted for persisted assistant messages",
);
}
const comment = input.comment?.trim() ? input.comment.trim() : null;
if (input.value === null) {
return { feedback: null };
}
const scoreId = `afbs_${input.messageId}_${ctx.session.user.id}`;
const now = new Date();
const scoreProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;
if (projectAvailability.aiTelemetryEnabled && scoreProjectId) {
// Approval continuations trace onto the root run, so score the root too.
const run = await ctx.prisma.inAppAgentRun.findUnique({
where: {
id_projectId: { id: input.runId, projectId: input.projectId },
},
select: { request: true },
});
const telemetryRunId = resolveInAppAgentRootRunId(
run?.request,
input.runId,
);
await upsertScore({
id: scoreId,
timestamp: convertDateToClickhouseDateTime(now),
project_id: scoreProjectId,
environment: IN_APP_AGENT_FEEDBACK_ENVIRONMENT,
trace_id: getInAppAgentInstrumentationTraceId(telemetryRunId),
observation_id:
getInAppAgentInstrumentationObservationId(telemetryRunId),
session_id: input.conversationId,
name: IN_APP_AGENT_FEEDBACK_SCORE_NAME,
value: input.value === "thumbs_up" ? 1 : 0,
source: ScoreSourceEnum.ANNOTATION,
comment,
author_user_id: ctx.session.user.id,
config_id: null,
data_type: ScoreDataTypeEnum.BOOLEAN,
string_value: input.value === "thumbs_up" ? "true" : "false",
queue_id: null,
created_at: convertDateToClickhouseDateTime(now),
updated_at: convertDateToClickhouseDateTime(now),
metadata: {
project_id: input.projectId,
conversation_id: input.conversationId,
message_id: input.messageId,
},
});
}
return { feedback: { value: input.value, comment } };
}),
/**
* The project this instance traces assistant turns into, where each
* conversation is a session. Internal surface only: null when the
* organization opted out of AI telemetry or tracing is not configured.
*/
telemetryProject: protectedProjectProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ ctx, input }) => {
if (
!hasInternalAccess({
isAdmin: ctx.session.user.admin === true,
isExperimentalFeaturesEnabled:
env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES === "true",
})
) {
throw new ForbiddenError(
"The assistant telemetry project is an internal surface.",
);
}
const projectAvailability = await assertInAppAgentAvailable({
prisma: ctx.prisma,
projectId: input.projectId,
user: ctx.session.user,
});
return {
projectId: projectAvailability.aiTelemetryEnabled
? (env.LANGFUSE_AI_FEATURES_PROJECT_ID ?? null)
: null,
};
}),
});