Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

22.8 KB
Raw Blame
import { z } from "zod";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
import {
  applyCommentFilters,
  traceException,
  getTracesTable,
  getTracesTableCount,
  getScoresForTraces,
  getNumericScoresGroupedByName,
  getBooleanScoresGroupedByName,
  getTracesGroupedByName,
  getTracesGroupedByTags,
  getObservationsForTrace,
  getTraceById,
  logger,
  upsertTrace,
  convertTraceDomainToClickhouse,
  hasAnyTracingData,
  traceDeletionProcessor,
  getTracesTableMetrics,
  getCategoricalScoresGroupedByName,
  convertDateToClickhouseDateTime,
  getAgentGraphData,
  tracesTableUiColumnDefinitions,
  getTracesGroupedByUsers,
  getTracesGroupedBySessionId,
  updateEvents,
  getScoresAndCorrectionsForTraces,
} from "@langfuse/shared/src/server";
import {
  createTRPCRouter,
  protectedGetTraceProcedure,
  protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
  isoDateTime,
  BatchActionQuerySchema,
  BatchTableNames,
  BatchExportTableName,
  BatchActionType,
  ActionId,
  filterAndValidateDbScoreList,
  normalizeOrderByForTable,
  orderBy,
  paginationZod,
  singleFilterList,
  timeFilter,
  type Observation,
  hasValidTracingSearchTypes,
  TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
  TracingSearchType,
  type ScoreDomain,
  ScoreDataTypeArray,
  ScoreDataTypeEnum,
  LISTABLE_SCORE_TYPES,
  LangfuseNotFoundError,
} from "@langfuse/shared";
import { TRPCError } from "@trpc/server";
import { createBatchActionJob } from "@/src/features/table/server/createBatchActionJob";
import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
import { sanitizeLegacyTracingSearch } from "@/src/features/traces/server/legacyIoSearch";
import {
  type AgentGraphDataResponse,
  AgentGraphDataSchema,
} from "@/src/features/trace-graph-view/types";

import { env } from "@/src/env.mjs";
import {
  toDomainWithStringifiedMetadata,
  toDomainArrayWithStringifiedMetadata,
} from "@/src/utils/clientSideDomainTypes";
import { scoreFilters } from "@/src/features/scores/lib/scoreColumns";
import partition from "lodash/partition";

const TraceCountOptions = z
  .object({
    projectId: z.string(), // Required for protectedProjectProcedure
    searchQuery: z.string().nullable(),
    searchType: z.array(TracingSearchType),
    filter: singleFilterList.nullable(),
    orderBy: orderBy,
  })
  .refine(hasValidTracingSearchTypes, {
    message: TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
    path: ["searchType"],
  });
const TraceFilterOptions = TraceCountOptions.safeExtend({
  ...paginationZod,
});
type TraceFilterOptions = z.infer<typeof TraceFilterOptions>;

export type ObservationReturnTypeWithMetadata = Omit<
  Observation,
  "input" | "output" | "metadata"
> & {
  traceId: string;
  metadata: string | null;
  traceName?: string | null;
  traceTags?: string[];
  // optional, because in v4 an observation can have those properties
  userId?: string | null;
  sessionId?: string | null;
  release?: string | null;
};

export type ObservationReturnType = Omit<
  ObservationReturnTypeWithMetadata,
  "metadata"
>;

export const traceRouter = createTRPCRouter({
  hasTracingConfigured: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      // Check if there is any tracing data in the database
      const hasTraces = await hasAnyTracingData(input.projectId);

      if (hasTraces) {
        return true;
      }

      // If no traces, check if data retention is configured
      // This indicates the user has configured tracing even if data retention cleaned all traces
      const project = await ctx.prisma.project.findUnique({
        where: {
          id: input.projectId,
        },
        select: {
          retentionDays: true,
        },
      });

      return !!(project?.retentionDays && project.retentionDays > 0);
    }),
  all: protectedProjectProcedure
    .input(TraceFilterOptions)
    .query(async ({ input, ctx }) => {
      const search = sanitizeLegacyTracingSearch({
        searchQuery: input.searchQuery,
        searchType: input.searchType,
        tableName: BatchTableNames.Traces,
      });

      const { filterState, hasNoMatches } = await applyCommentFilters({
        filterState: input.filter ?? [],
        prisma: ctx.prisma,
        projectId: ctx.session.projectId,
        objectType: "TRACE",
      });

      if (hasNoMatches) {
        return { traces: [] };
      }

      const traces = await getTracesTable({
        projectId: ctx.session.projectId,
        filter: filterState,
        searchQuery: search.searchQuery,
        searchType: search.searchType ?? ["id"],
        orderBy: normalizeOrderByForTable({
          orderBy: input.orderBy,
          expectedTimeColumn: "timestamp",
        }),
        limit: input.limit,
        page: input.page,
      });
      return { traces };
    }),
  countAll: protectedProjectProcedure
    .input(TraceCountOptions)
    .query(async ({ input, ctx }) => {
      const search = sanitizeLegacyTracingSearch({
        searchQuery: input.searchQuery,
        searchType: input.searchType,
        tableName: BatchTableNames.Traces,
      });

      const { filterState, hasNoMatches } = await applyCommentFilters({
        filterState: input.filter ?? [],
        prisma: ctx.prisma,
        projectId: ctx.session.projectId,
        objectType: "TRACE",
      });

      if (hasNoMatches) {
        return { totalCount: 0 };
      }

      const count = await getTracesTableCount({
        projectId: ctx.session.projectId,
        filter: filterState,
        searchType: search.searchType ?? ["id"],
        searchQuery: search.searchQuery,
        limit: 1,
        page: 0,
      });

      return {
        totalCount: count,
      };
    }),
  metrics: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        traceIds: z.array(z.string()),
        filter: singleFilterList.nullable(),
      }),
    )
    .query(async ({ input, ctx }) => {
      if (input.traceIds.length === 0) return [];

      const { filterState, hasNoMatches, matchingIds } =
        await applyCommentFilters({
          filterState: input.filter ?? [],
          prisma: ctx.prisma,
          projectId: ctx.session.projectId,
          objectType: "TRACE",
        });

      if (hasNoMatches) {
        return [];
      }

      // If comment filters returned matching IDs, intersect with input.traceIds
      let filteredTraceIds = input.traceIds;
      if (matchingIds !== null) {
        filteredTraceIds = input.traceIds.filter((id) =>
          matchingIds.includes(id),
        );

        if (filteredTraceIds.length === 0) {
          return [];
        }
      }

      // Remove the comment filter's ID injection and use filteredTraceIds instead
      const filterWithoutCommentIds = filterState.filter(
        (f) =>
          !(
            f.type === "stringOptions" &&
            f.column === "id" &&
            f.operator === "any of"
          ),
      );

      const res = await getTracesTableMetrics({
        projectId: ctx.session.projectId,
        filter: [
          ...filterWithoutCommentIds,
          {
            type: "stringOptions",
            operator: "any of",
            column: "ID",
            value: filteredTraceIds,
          },
        ],
        orderBy: { column: "timestamp", order: "DESC" },
      });

      const traceScores = await getScoresForTraces({
        projectId: ctx.session.projectId,
        traceIds: res.map((r) => r.id),
        excludeMetadata: true,
        includeHasMetadata: true,
      });

      const validatedScores = filterAndValidateDbScoreList({
        scores: traceScores,
        dataTypes: LISTABLE_SCORE_TYPES,
        includeHasMetadata: true,
        onParseError: traceException,
      });

      return res.map((row) => ({
        ...row,
        scores: aggregateScores(
          validatedScores.filter((s) => s.traceId === row.id),
        ),
      }));
    }),
  filterOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        timestampFilter: z.array(timeFilter).optional(),
      }),
    )
    .query(async ({ input }) => {
      const { timestampFilter } = input;
      // Trace table filters/columns operate on trace-scoped aggregates: any
      // score row attached to the trace, whether it lives on the trace itself
      // or on one of the trace's observations.
      const traceScopedScoreFilters = [
        ...(timestampFilter ?? []),
        ...scoreFilters.forTraceScopedAggregates(),
      ];

      const [
        numericScoreNames,
        categoricalScoreNames,
        booleanScoreNames,
        traceNames,
        tags,
        userIds,
        sessionIds,
      ] = await Promise.all([
        getNumericScoresGroupedByName(input.projectId, traceScopedScoreFilters),
        getCategoricalScoresGroupedByName(
          input.projectId,
          traceScopedScoreFilters,
        ),
        getBooleanScoresGroupedByName(input.projectId, traceScopedScoreFilters),
        getTracesGroupedByName(
          input.projectId,
          tracesTableUiColumnDefinitions,
          timestampFilter ?? [],
        ),
        getTracesGroupedByTags({
          projectId: input.projectId,
          filter: timestampFilter ?? [],
        }),
        getTracesGroupedByUsers(
          input.projectId,
          timestampFilter ?? [],
          undefined,
          100,
          0,
        ),
        getTracesGroupedBySessionId(
          input.projectId,
          timestampFilter ?? [],
          undefined,
          100,
          0,
        ),
      ]);

      return {
        name: traceNames.map((n) => ({ value: n.name, count: n.count })),
        scores_avg: numericScoreNames.map((s) => s.name),
        score_categories: categoricalScoreNames,
        score_booleans: booleanScoreNames.map((s) => s.name),
        tags: tags,
        users: userIds.map((u) => ({
          value: u.user,
          count: u.count,
        })),
        sessions: sessionIds.map((s) => ({
          value: s.session_id,
          count: s.count,
        })),
      };
    }),
  byId: protectedGetTraceProcedure
    .input(
      z.object({
        traceId: z.string(), // used for security check
        projectId: z.string(), // used for security check
        timestamp: z.date().nullish(), // timestamp of the trace. Used to query CH more efficiently
        fromTimestamp: z.date().nullish(), // min timestamp of the trace. Used to query CH more efficiently
        verbosity: z.enum(["compact", "truncated", "full"]).default("full"),
      }),
    )
    .query(async ({ ctx }) => {
      if (!ctx.trace) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Trace not found",
        });
      }
      return {
        ...ctx.trace,
        input: ctx.trace.input as string,
        output: ctx.trace.output as string,
        metadata: ctx.trace.metadata
          ? JSON.stringify(ctx.trace.metadata)
          : undefined,
      };
    }),
  byIdWithObservationsAndScores: protectedGetTraceProcedure
    .input(
      z.object({
        traceId: z.string(), // used for security check
        timestamp: z.date().nullish(), // timestamp of the trace. Used to query CH more efficiently
        fromTimestamp: z.date().nullish(), // min timestamp of the trace. Used to query CH more efficiently
        projectId: z.string(), // used for security check
      }),
    )
    .query(async ({ input, ctx }) => {
      if (!ctx.trace) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Trace not found",
        });
      }

      const [observations, traceScores] = await Promise.all([
        getObservationsForTrace({
          traceId: input.traceId,
          projectId: input.projectId,
          timestamp: input.timestamp ?? input.fromTimestamp ?? undefined,
          includeIO: false,
        }),
        getScoresAndCorrectionsForTraces({
          projectId: input.projectId,
          traceIds: [input.traceId],
          timestamp: input.timestamp ?? input.fromTimestamp ?? undefined,
        }),
      ]);

      const validatedScores = filterAndValidateDbScoreList({
        scores: traceScores,
        dataTypes: [...ScoreDataTypeArray],
        onParseError: traceException,
      });

      const [corrections, scores] = partition(
        validatedScores,
        (s) => s.dataType === ScoreDataTypeEnum.CORRECTION,
      );

      const obsStartTimes = observations
        .map((o) => o.startTime)
        .sort((a, b) => a.getTime() - b.getTime());
      const obsEndTimes = observations
        .map((o) => o.endTime)
        .filter((t) => t)
        .sort((a, b) => (a as Date).getTime() - (b as Date).getTime());
      const latencyMs = (() => {
        if (obsStartTimes.length > 0) {
          if (obsEndTimes.length > 0) {
            return (
              (obsEndTimes[obsEndTimes.length - 1] as Date).getTime() -
              obsStartTimes[0]!.getTime()
            );
          }
          if (obsStartTimes.length > 1) {
            return (
              obsStartTimes[obsStartTimes.length - 1]!.getTime() -
              obsStartTimes[0]!.getTime()
            );
          }
          return undefined;
        }
        return undefined;
      })();

      const scoresDomain =
        toDomainArrayWithStringifiedMetadata<ScoreDomain>(scores);

      return {
        ...toDomainWithStringifiedMetadata(ctx.trace),
        input: ctx.trace.input ? JSON.stringify(ctx.trace.input) : null,
        output: ctx.trace.output ? JSON.stringify(ctx.trace.output) : null,
        scores: scoresDomain,
        corrections,
        latency: latencyMs !== undefined ? latencyMs / 1000 : undefined,
        observations: observations.map((o) => ({
          ...toDomainWithStringifiedMetadata(o),
          output: undefined,
          input: undefined, // this is not queried above.
        })) as ObservationReturnTypeWithMetadata[],
      };
    }),
  deleteMany: protectedProjectProcedure
    .input(
      z
        .object({
          traceIds: z.array(z.string()),
          projectId: z.string(),
          query: BatchActionQuerySchema.optional(),
          isBatchAction: z.boolean().default(false),
        })
        // Batch actions delete by query and ignore traceIds, so an empty list
        // is valid there (paging/refetch can drain the visible selection while
        // select-all is armed); only id-based deletes need at least one id.
        .refine((input) => input.isBatchAction || input.traceIds.length > 0, {
          message: "Minimum 1 traceId is required.",
          path: ["traceIds"],
        })
        .refine((input) => !input.isBatchAction || input.query !== undefined, {
          message: "Batch actions require a query.",
          path: ["query"],
        }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "traces:delete",
      });

      throwIfNoEntitlement({
        entitlement: "trace-deletion",
        projectId: input.projectId,
        sessionUser: ctx.session.user,
      });

      if (input.isBatchAction && input.query) {
        // Comment filters (commentCount/commentContent in both the v3 traces
        // and v4 events views) resolve via Postgres lookups at read time
        // (applyCommentFilters); when the worker translates the stored
        // filters into ClickHouse SQL, these columns map to a nonexistent
        // "comments" table and would deterministically fail every batch, so
        // reject them at dispatch.
        const hasCommentFilter = (input.query.filter ?? []).some(
          (f) => f.column === "commentCount" || f.column === "commentContent",
        );
        if (hasCommentFilter) {
          throw new TRPCError({
            code: "BAD_REQUEST",
            message:
              "Batch deletion does not support comment filters. Remove the comment filter and try again.",
          });
        }

        // Decide here whether this delete reads from the events table: the
        // v4 events view sets query.useEventsTable: true, which we honor
        // after checking the events view is actually available to this user
        // (v4 beta flag, or the instance-wide preview opt-in). In every
        // other case createBatchActionJob infers the choice from the user's
        // v4 beta flag.
        const declaresEventsTable = input.query.useEventsTable === true;
        if (declaresEventsTable) {
          const eventsSurfaceAvailable =
            ctx.session.user.v4BetaEnabled === true ||
            env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true";
          if (!eventsSurfaceAvailable) {
            throw new TRPCError({
              code: "BAD_REQUEST",
              message:
                "Events-backed batch deletion is not available for this user on this instance.",
            });
          }
        }

        await createBatchActionJob({
          projectId: input.projectId,
          actionId: ActionId.TraceDelete,
          actionType: BatchActionType.Delete,
          tableName: BatchExportTableName.Traces,
          session: ctx.session,
          query: input.query,
          useEventsTableOverride: declaresEventsTable ? true : undefined,
        });
      } else {
        await Promise.all(
          input.traceIds.map((traceId) =>
            auditLog({
              resourceType: "trace",
              resourceId: traceId,
              action: "delete",
              session: ctx.session,
            }),
          ),
        );

        await traceDeletionProcessor(input.projectId, input.traceIds, {
          actor: { type: "USER", userId: ctx.session.user.id },
        });
      }
    }),
  bookmark: protectedProjectProcedure
    .input(
      z.object({
        traceId: z.string(),
        projectId: z.string(),
        bookmarked: z.boolean(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "objects:bookmark",
      });
      try {
        await auditLog({
          session: ctx.session,
          resourceType: "trace",
          resourceId: input.traceId,
          action: "bookmark",
          after: input.bookmarked,
        });

        let trace;

        // eslint-disable-next-line @typescript-eslint/no-deprecated
        const clickhouseTrace = await getTraceById({
          traceId: input.traceId,
          projectId: input.projectId,
        });
        if (clickhouseTrace) {
          trace = clickhouseTrace;
          clickhouseTrace.bookmarked = input.bookmarked;
          const promises = [
            upsertTrace(convertTraceDomainToClickhouse(clickhouseTrace)),
          ];
          if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy") {
            promises.push(
              updateEvents(
                input.projectId,
                { traceIds: [clickhouseTrace.id], rootOnly: true },
                { bookmarked: input.bookmarked },
              ),
            );
          }
          await Promise.all(promises);
        } else {
          logger.error(
            `Trace not found in Clickhouse: ${input.traceId}. Skipping bookmark.`,
          );
        }

        return trace;
      } catch (error) {
        logger.error("Failed to call traces.bookmark", error);
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
        });
      }
    }),
  publish: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        traceId: z.string(),
        public: z.boolean(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "objects:publish",
      });
      await auditLog({
        session: ctx.session,
        resourceType: "trace",
        resourceId: input.traceId,
        action: "publish",
        after: input.public,
      });

      // eslint-disable-next-line @typescript-eslint/no-deprecated
      const clickhouseTrace = await getTraceById({
        traceId: input.traceId,
        projectId: input.projectId,
      });
      if (!clickhouseTrace) {
        throw new LangfuseNotFoundError("Trace not found");
      }
      clickhouseTrace.public = input.public;
      const promises = [
        upsertTrace(convertTraceDomainToClickhouse(clickhouseTrace)),
      ];
      if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy") {
        promises.push(
          updateEvents(
            input.projectId,
            { traceIds: [clickhouseTrace.id] },
            { public: input.public },
          ),
        );
      }
      await Promise.all(promises);
      return clickhouseTrace;
    }),
  getAgentGraphData: protectedGetTraceProcedure
    .input(
      z.object({
        projectId: z.string(),
        traceId: z.string(),
        minStartTime: isoDateTime({ offset: true }),
        maxStartTime: isoDateTime({ offset: true }),
        // Optional fields for enforceTraceAccess middleware (supports public traces)
        timestamp: z.date().nullish(),
        fromTimestamp: z.date().nullish(),
      }),
    )
    .query(async ({ input }): Promise<Required<AgentGraphDataResponse>[]> => {
      const { traceId, projectId, minStartTime, maxStartTime } = input;

      const chMinStartTime = convertDateToClickhouseDateTime(
        new Date(minStartTime),
      );
      const chMaxStartTime = convertDateToClickhouseDateTime(
        new Date(maxStartTime),
      );

      const records = await getAgentGraphData({
        projectId,
        traceId,
        chMinStartTime,
        chMaxStartTime,
      });

      const result = records
        .map((r) => {
          const parsed = AgentGraphDataSchema.safeParse(r);
          if (!parsed.success) {
            return null;
          }

          const data = parsed.data;
          const hasLangGraphData = data.step != null && data.node != null;
          const hasAgentData = data.type !== "EVENT"; // Include all types except EVENT

          if (hasLangGraphData) {
            return {
              id: data.id,
              node: data.node,
              step: data.step,
              parentObservationId: data.parent_observation_id || null,
              name: data.name,
              startTime: data.start_time,
              endTime: data.end_time || undefined,
              observationType: data.type,
            };
          } else if (hasAgentData) {
            return {
              id: data.id,
              node: data.name,
              step: 0,
              parentObservationId: data.parent_observation_id || null,
              name: data.name,
              startTime: data.start_time,
              endTime: data.end_time || undefined,
              observationType: data.type,
            };
          }

          return null;
        })
        .filter((r) => Boolean(r)) as Required<AgentGraphDataResponse>[];

      return result;
    }),
});