Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

15.8 KB
Raw Blame
import { z } from "zod";

import {
  ForbiddenError,
  InvalidRequestError,
  resolveInAppAgentRootRunId,
  ScoreDataTypeEnum,
  ScoreSourceEnum,
  TEXT_SCORE_MAX_LENGTH,
} from "@langfuse/shared";
import {
  convertDateToClickhouseDateTime,
  upsertScore,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import {
  AgUiContextSchema,
  getInAppAgentInstrumentationObservationId,
  getInAppAgentInstrumentationTraceId,
  IN_APP_AGENT_PRODUCT_ENVIRONMENT,
  InAppAgentToolInputsSchema,
} from "@langfuse/shared/in-app-agent";
import { InAppAgentMessageFeedbackValueSchema } from "../schema";
import {
  assertInAppAgentAvailable,
  assertInAppAgentModelConfigured,
} from "@/src/features/in-app-agent/server/availability";
import {
  createTRPCRouter,
  protectedProjectProcedure,
  protectedProjectProcedureWithoutTracing,
} from "@/src/server/api/trpc";
import {
  getConversationMessages,
  getOwnedConversationOrThrow,
  serializeConversation,
} from "@langfuse/shared/in-app-agent/server/persistence";
import {
  assertInAppAgentRateLimit,
  getInAppAgentApiAccessScope,
} from "@/src/features/in-app-agent/server/rateLimit";
import { assertInAppAgentRunCapacity } from "@/src/features/in-app-agent/server/runCapacity";
import { hasInternalAccess } from "@/src/features/feature-flags/utils";
import {
  cancelBackgroundRun,
  decideBackgroundApproval,
  deleteBackgroundConversation,
  getBackgroundConversationSnapshot,
  serializeConversationLatestRun,
  startBackgroundRun,
} from "@/src/features/in-app-agent/server/backgroundRunService";

const CONVERSATION_LIST_LIMIT = 50;
const MAX_IN_APP_AGENT_MESSAGE_LENGTH = 32_000;

const ConversationListCursorSchema = z.object({
  updatedAt: z.date(),
  id: z.string(),
});

const ConversationIdInput = z.object({
  projectId: z.string(),
  conversationId: z.string(),
});

const RenameConversationInput = ConversationIdInput.extend({
  title: z.string().trim().min(1).max(80),
});

const StartRunInput = ConversationIdInput.extend({
  message: z.string().trim().min(1).max(MAX_IN_APP_AGENT_MESSAGE_LENGTH),
  /**
   * The AG-UI context for the current page, quick action, and entry point;
   * resolved and sanitized server-side, then stored for the worker to replay.
   */
  context: z.array(AgUiContextSchema).default([]),
});

const CancelRunInput = ConversationIdInput.extend({
  runId: z.string(),
});

const DecideToolApprovalInput = ConversationIdInput.extend({
  runId: z.string(),
  toolCallId: z.string(),
  approved: z.boolean(),
  // "conversation" also approves this call; it never means approve-without-run.
  approvalScope: z.enum(["once", "conversation"]).default("once"),
  /** Validated against the persisted tool call in the service. */
  toolInputs: InAppAgentToolInputsSchema.optional(),
})
  .refine((input) => input.approved || input.approvalScope === "once", {
    message: "A rejection cannot grant a tool",
    path: ["approvalScope"],
  })
  .refine((input) => input.approved || input.toolInputs === undefined, {
    message: "A rejection cannot carry tool inputs",
    path: ["toolInputs"],
  });

const SubmitFeedbackInput = ConversationIdInput.extend({
  messageId: z.string(),
  runId: z.string(),
  value: InAppAgentMessageFeedbackValueSchema.nullable(),
  comment: z.string().trim().max(TEXT_SCORE_MAX_LENGTH).nullable().optional(),
});

const IN_APP_AGENT_FEEDBACK_SCORE_NAME = "in_app_agent_feedback";
const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = IN_APP_AGENT_PRODUCT_ENVIRONMENT;

export const inAppAgentRouter = createTRPCRouter({
  // Each row carries its newest run summary for the activity poll and badges.
  // Activity clients request the newest page only (see
  // IN_APP_AGENT_ACTIVITY_LIST_LIMIT); quieter conversations outside that
  // window are an accepted attention gap for now.
  listConversations: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        cursor: ConversationListCursorSchema.optional(),
        limit: z.number().int().min(1).max(CONVERSATION_LIST_LIMIT).default(50),
      }),
    )
    .query(async ({ ctx, input }) => {
      await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      const conversations = await ctx.prisma.inAppAgentConversation.findMany({
        where: {
          projectId: input.projectId,
          createdByUserId: ctx.session.user.id,
          deletedAt: null,
          ...(input.cursor
            ? {
                OR: [
                  { updatedAt: { lt: input.cursor.updatedAt } },
                  {
                    updatedAt: input.cursor.updatedAt,
                    id: { lt: input.cursor.id },
                  },
                ],
              }
            : {}),
        },
        orderBy: [{ updatedAt: "desc" }, { id: "desc" }],
        take: input.limit + 1,
        relationLoadStrategy: "join",
        select: {
          id: true,
          title: true,
          createdAt: true,
          updatedAt: true,
          runs: {
            orderBy: { createdAt: "desc" },
            take: 1,
            select: {
              id: true,
              status: true,
              errorCode: true,
              cancelRequestedAt: true,
              createdAt: true,
              claimedAt: true,
              heartbeatAt: true,
              finishedAt: true,
            },
          },
        },
      });

      const page = conversations.slice(0, input.limit);
      const lastConversation = page.at(-1);

      return {
        conversations: page.map((conversation) => ({
          ...serializeConversation(conversation),
          latestRun: serializeConversationLatestRun(conversation.runs[0]),
        })),
        nextCursor:
          conversations.length > input.limit && lastConversation
            ? {
                updatedAt: lastConversation.updatedAt,
                id: lastConversation.id,
              }
            : undefined,
      };
    }),

  getConversation: protectedProjectProcedureWithoutTracing
    .input(ConversationIdInput)
    .query(async ({ ctx, input }) => {
      await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      return getBackgroundConversationSnapshot({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        userId: ctx.session.user.id,
      });
    }),

  /**
   * Submit a turn for background execution.
   *
   * Validates the request, then commits the run as QUEUED with its user message
   * already appended — the events table is
   * the render source from the instant of submit, so there is no optimistic UI
   * state to survive a refresh. The BullMQ enqueue happens after commit; a
   * failure there marks the run FAILED immediately rather than leaving a
   * committed run nobody will execute.
   */
  startRun: protectedProjectProcedureWithoutTracing
    .input(StartRunInput)
    .mutation(async ({ ctx, input }) => {
      const projectAvailability = await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });
      const modelConfig = assertInAppAgentModelConfigured();

      const rateLimitScope = getInAppAgentApiAccessScope(
        ctx.session.user,
        input.projectId,
        projectAvailability,
      );

      await assertInAppAgentRateLimit(rateLimitScope, "in-app-agent-run");

      // The concurrency ceiling is enforced inside `startBackgroundRun`, which
      // owns the conversation lookup it has to reconcile against first.
      return startBackgroundRun({
        prisma: ctx.prisma,
        projectId: input.projectId,
        orgId: rateLimitScope.orgId,
        plan: rateLimitScope.plan,
        conversationId: input.conversationId,
        userId: ctx.session.user.id,
        message: input.message,
        context: input.context,
        isV4Enabled: ctx.session.user.v4BetaEnabled === true,
        model: modelConfig.modelId,
        aiTelemetryEnabled: projectAvailability.aiTelemetryEnabled,
      });
    }),

  cancelRun: protectedProjectProcedureWithoutTracing
    .input(CancelRunInput)
    .mutation(async ({ ctx, input }) => {
      await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      return cancelBackgroundRun({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        runId: input.runId,
        userId: ctx.session.user.id,
      });
    }),

  /**
   * Decide a pending tool approval.
   *
   * The client sends only IDs and a boolean. The tool name and arguments are
   * read server-side from the persisted interrupt event, so there is nothing
   * to tamper with on the way back and no fingerprint to keep in sync.
   */
  decideToolApproval: protectedProjectProcedureWithoutTracing
    .input(DecideToolApprovalInput)
    .mutation(async ({ ctx, input }) => {
      const projectAvailability = await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });
      const modelConfig = assertInAppAgentModelConfigured();

      const rateLimitScope = getInAppAgentApiAccessScope(
        ctx.session.user,
        input.projectId,
        projectAvailability,
      );

      await assertInAppAgentRateLimit(rateLimitScope, "in-app-agent-run");
      // A continuation is a fresh run that occupies a worker slot, and parked
      // approvals accumulate without holding one, so this path needs the
      // ceiling for the same reason it already needs the daily bucket.
      await assertInAppAgentRunCapacity({
        prisma: ctx.prisma,
        orgId: rateLimitScope.orgId,
        plan: rateLimitScope.plan,
        userId: ctx.session.user.id,
      });

      return decideBackgroundApproval({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        runId: input.runId,
        toolCallId: input.toolCallId,
        approved: input.approved,
        approvalScope: input.approvalScope,
        toolInputs: input.toolInputs,
        userId: ctx.session.user.id,
        model: modelConfig.modelId,
      });
    }),

  deleteConversation: protectedProjectProcedureWithoutTracing
    .input(ConversationIdInput)
    .mutation(async ({ ctx, input }) => {
      await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      return deleteBackgroundConversation({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        userId: ctx.session.user.id,
      });
    }),

  renameConversation: protectedProjectProcedureWithoutTracing
    .input(RenameConversationInput)
    .mutation(async ({ ctx, input }) => {
      await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      await getOwnedConversationOrThrow({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        userId: ctx.session.user.id,
      });

      const conversation = await ctx.prisma.inAppAgentConversation.update({
        where: {
          id_projectId: {
            id: input.conversationId,
            projectId: input.projectId,
          },
          createdByUserId: ctx.session.user.id,
          deletedAt: null,
        },
        data: {
          title: input.title,
          renamedByUserAt: new Date(),
        },
      });

      return { conversation: serializeConversation(conversation) };
    }),

  submitFeedback: protectedProjectProcedureWithoutTracing
    .input(SubmitFeedbackInput)
    .mutation(async ({ ctx, input }) => {
      const projectAvailability = await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      await getOwnedConversationOrThrow({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
        userId: ctx.session.user.id,
      });

      const messages = await getConversationMessages({
        prisma: ctx.prisma,
        projectId: input.projectId,
        conversationId: input.conversationId,
      });
      const targetMessage = messages.find(
        (message) => message.id === input.messageId,
      );

      if (
        targetMessage?.role !== "assistant" ||
        typeof targetMessage.content !== "string" ||
        targetMessage.content.length === 0
      ) {
        throw new InvalidRequestError(
          "Feedback can only be submitted for assistant text messages",
        );
      }

      if (targetMessage.runId !== input.runId) {
        throw new InvalidRequestError(
          "Feedback can only be submitted for persisted assistant messages",
        );
      }

      const comment = input.comment?.trim() ? input.comment.trim() : null;
      if (input.value === null) {
        return { feedback: null };
      }

      const scoreId = `afbs_${input.messageId}_${ctx.session.user.id}`;
      const now = new Date();
      const scoreProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;

      if (projectAvailability.aiTelemetryEnabled && scoreProjectId) {
        // Approval continuations trace onto the root run, so score the root too.
        const run = await ctx.prisma.inAppAgentRun.findUnique({
          where: {
            id_projectId: { id: input.runId, projectId: input.projectId },
          },
          select: { request: true },
        });
        const telemetryRunId = resolveInAppAgentRootRunId(
          run?.request,
          input.runId,
        );

        await upsertScore({
          id: scoreId,
          timestamp: convertDateToClickhouseDateTime(now),
          project_id: scoreProjectId,
          environment: IN_APP_AGENT_FEEDBACK_ENVIRONMENT,
          trace_id: getInAppAgentInstrumentationTraceId(telemetryRunId),
          observation_id:
            getInAppAgentInstrumentationObservationId(telemetryRunId),
          session_id: input.conversationId,
          name: IN_APP_AGENT_FEEDBACK_SCORE_NAME,
          value: input.value === "thumbs_up" ? 1 : 0,
          source: ScoreSourceEnum.ANNOTATION,
          comment,
          author_user_id: ctx.session.user.id,
          config_id: null,
          data_type: ScoreDataTypeEnum.BOOLEAN,
          string_value: input.value === "thumbs_up" ? "true" : "false",
          queue_id: null,
          created_at: convertDateToClickhouseDateTime(now),
          updated_at: convertDateToClickhouseDateTime(now),
          metadata: {
            project_id: input.projectId,
            conversation_id: input.conversationId,
            message_id: input.messageId,
          },
        });
      }

      return { feedback: { value: input.value, comment } };
    }),

  /**
   * The project this instance traces assistant turns into, where each
   * conversation is a session. Internal surface only: null when the
   * organization opted out of AI telemetry or tracing is not configured.
   */
  telemetryProject: protectedProjectProcedure
    .input(z.object({ projectId: z.string() }))
    .query(async ({ ctx, input }) => {
      if (
        !hasInternalAccess({
          isAdmin: ctx.session.user.admin === true,
          isExperimentalFeaturesEnabled:
            env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES === "true",
        })
      ) {
        throw new ForbiddenError(
          "The assistant telemetry project is an internal surface.",
        );
      }

      const projectAvailability = await assertInAppAgentAvailable({
        prisma: ctx.prisma,
        projectId: input.projectId,
        user: ctx.session.user,
      });

      return {
        projectId: projectAvailability.aiTelemetryEnabled
          ? (env.LANGFUSE_AI_FEATURES_PROJECT_ID ?? null)
          : null,
      };
    }),
});