🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: trace the real cause of tRPC 5xx errors and nest Datadog log trace ids (#18546)
Niklas Semmler911c91ad7f4b
diff --git a/packages/shared/src/server/logger.test.ts b/packages/shared/src/server/logger.test.tsindex 09f285dde5..235f6c74fa 100644--- a/packages/shared/src/server/logger.test.ts+++ b/packages/shared/src/server/logger.test.ts@@ -8,6 +8,9 @@ vi.mock("../env", () => ({},}));+const getCurrentSpan = vi.hoisted(() => vi.fn());+vi.mock("./instrumentation", () => ({ getCurrentSpan }));+import { logger } from "./logger";const jsonLine = (meta: Record<string, unknown>) => {@@ -65,3 +68,25 @@ describe("logger JSON format", () => {});});});++describe("logger trace correlation", () => {+ it("nests the Datadog ids under dd and keeps the hex OTel ids", () => {+ getCurrentSpan.mockReturnValueOnce({+ spanContext: () => ({+ traceId: "617222771252365135e4848d739614a8",+ spanId: "00f067aa0ba902b7",+ }),+ });++ const line = jsonLine({});++ expect(line.dd).toEqual({+ trace_id: "3883374521764680872",+ span_id: "67667974448284343",+ });+ expect(line).not.toHaveProperty(["dd.trace_id"]);+ expect(line).not.toHaveProperty(["dd.span_id"]);+ expect(line.trace_id).toBe("617222771252365135e4848d739614a8");+ expect(line.span_id).toBe("00f067aa0ba902b7");+ });+});diff --git a/packages/shared/src/server/logger.ts b/packages/shared/src/server/logger.tsindex 1979b274f0..2138781d0e 100644--- a/packages/shared/src/server/logger.ts+++ b/packages/shared/src/server/logger.ts@@ -9,8 +9,10 @@ const tracingFormat = function () {if (span) {const { spanId, traceId } = span.spanContext();const traceIdEnd = traceId.slice(traceId.length / 2);- info["dd.trace_id"] = BigInt(`0x${traceIdEnd}`).toString();- info["dd.span_id"] = BigInt(`0x${spanId}`).toString();+ info.dd = {+ trace_id: BigInt(`0x${traceIdEnd}`).toString(),+ span_id: BigInt(`0x${spanId}`).toString(),+ };info["trace_id"] = traceId;info["span_id"] = spanId;}diff --git a/web/src/__tests__/server/trpc-error-formatting.servertest.ts b/web/src/__tests__/server/trpc-error-formatting.servertest.tsindex 5d013095a5..77e933314f 100644--- a/web/src/__tests__/server/trpc-error-formatting.servertest.ts+++ b/web/src/__tests__/server/trpc-error-formatting.servertest.ts@@ -6,12 +6,20 @@ vi.mock("@langfuse/shared/src/server", async () => ({error: vi.fn(),debug: vi.fn(),},+ traceException: vi.fn(),}));import type { Session } from "next-auth";import { TRPCError } from "@trpc/server";import * as z from "zod";-import { ClickHouseResourceError, logger } from "@langfuse/shared/src/server";+import {+ ClickHouseResourceError,+ logger,+ traceException,+} from "@langfuse/shared/src/server";+import { Prisma } from "@langfuse/shared/src/db";+import { fetchRequestHandler } from "@trpc/server/adapters/fetch";+import { reportTRPCError } from "@/src/server/utils/trpc-utils";import {createInnerTRPCContext,createTRPCRouter,@@ -130,4 +138,64 @@ describe("tRPC error formatting", () => {expect(formattedWithStack.data["stack"]).toBe("dev stack");});++ describe("over HTTP", () => {+ const fetchFailing = (thrown: unknown) => {+ const router = createTRPCRouter({+ failing: protectedProcedureWithoutTracing.query(() => {+ throw thrown;+ }),+ });+ return fetchRequestHandler({+ endpoint: "/api/trpc",+ req: new Request("http://localhost/api/trpc/failing"),+ router,+ createContext: () =>+ createInnerTRPCContext({+ session: { user: { id: "user-1" } } as Session,+ headers: {},+ }),+ onError: reportTRPCError,+ });+ };++ it.each([+ [+ "Prisma",+ new Prisma.PrismaClientKnownRequestError(+ "Unique constraint failed on the fields: (`id`)",+ { code: "P2002", clientVersion: "test" },+ ),+ ],+ [+ "uncaused TRPCError",+ new TRPCError({+ code: "INTERNAL_SERVER_ERROR",+ message: "connection refused",+ }),+ ],+ ])(+ "traces the %s cause on the span but returns a generic 5xx",+ async (_, original) => {+ const res = await fetchFailing(original);+ const body = await res.text();++ expect(res.status).toBe(500);+ expect(body).toContain("Internal error. ");+ expect(body).not.toContain(original.message);+ expect(traceException).toHaveBeenCalledTimes(1);+ expect(vi.mocked(traceException).mock.calls[0]?.[0]).toBe(original);+ },+ );++ it("keeps the message of a 4xx error and does not trace it", async () => {+ const res = await fetchFailing(+ new TRPCError({ code: "BAD_REQUEST", message: "Invalid filter" }),+ );++ expect(res.status).toBe(400);+ expect(await res.text()).toContain("Invalid filter");+ expect(traceException).not.toHaveBeenCalled();+ });+ });});diff --git a/web/src/pages/api/trpc/[trpc].ts b/web/src/pages/api/trpc/[trpc].tsindex 573a253926..d0cafa3772 100644--- a/web/src/pages/api/trpc/[trpc].ts+++ b/web/src/pages/api/trpc/[trpc].ts@@ -2,8 +2,7 @@ import { createNextApiHandler } from "@trpc/server/adapters/next";import { createTRPCContext } from "@/src/server/api/trpc";import { appRouter } from "@/src/server/api/root";import { env } from "@/src/env.mjs";-import { logger, traceException } from "@langfuse/shared/src/server";-import { getTRPCErrorReporting } from "@/src/server/utils/trpc-utils";+import { reportTRPCError } from "@/src/server/utils/trpc-utils";export const config = {maxDuration: 240,@@ -25,24 +24,7 @@ export default createNextApiHandler({// option), but it only widens the accepted method for queries (read-only);// mutations remain POST-only, so the GET-mutation protection is unchanged.allowMethodOverride: true,- onError: ({ path, error }) => {- const { logLevel, shouldTrace } = getTRPCErrorReporting(error);- const message = `tRPC route failed on ${path ?? "<no-path>"}: ${error.message}`;-- if (logLevel === "error") {- logger.error(message, error);- } else if (logLevel === "warn") {- logger.warn(message, error);- } else {- logger.info(message, error);- }-- if (shouldTrace) {- traceException(error);- }-- return error;- },+ onError: reportTRPCError,responseMeta() {return {headers: {diff --git a/web/src/server/api/trpc.ts b/web/src/server/api/trpc.tsindex ee0fe57334..a57445fcc4 100644--- a/web/src/server/api/trpc.ts+++ b/web/src/server/api/trpc.ts@@ -23,7 +23,10 @@ import { prisma, Role } from "@langfuse/shared/src/db";import * as z from "zod";import * as opentelemetry from "@opentelemetry/api";import { type IncomingHttpHeaders } from "node:http";-import { getTRPCErrorCodeFromHTTPStatusCode } from "@/src/server/utils/trpc-utils";+import {+ createScrubbedError,+ getTRPCErrorCodeFromHTTPStatusCode,+} from "@/src/server/utils/trpc-utils";import { sendAdminAccessWebhook } from "@/src/server/adminAccessWebhook";type CreateContextOptions = {@@ -217,13 +220,19 @@ const withErrorHandling = t.middleware(async ({ ctx, next }) => {: "Please check error logs in your self-hosted deployment.";logErrorByStatus({ errorCode: code, httpStatus, error: res.error });- res.error = new TRPCError({- code,- cause: null, // do not expose stack traces- message: isSafeToExpose- ? res.error.message- : "Internal error. " + errorMessage,- });+ if (isSafeToExpose) {+ res.error = new TRPCError({+ code,+ cause: null, // do not expose stack traces+ message: res.error.message,+ });+ } else {+ res.error = createScrubbedError({+ code,+ message: "Internal error. " + errorMessage,+ original: res.error.cause ?? res.error,+ });+ }}}diff --git a/web/src/server/utils/trpc-utils.ts b/web/src/server/utils/trpc-utils.tsindex 7f3c52c060..91f0b6b8d7 100644--- a/web/src/server/utils/trpc-utils.ts+++ b/web/src/server/utils/trpc-utils.ts@@ -1,5 +1,6 @@-import type { TRPCError, TRPC_ERROR_CODE_KEY } from "@trpc/server";+import { TRPCError, type TRPC_ERROR_CODE_KEY } from "@trpc/server";import { getHTTPStatusCodeFromError } from "@trpc/server/http";+import { logger, traceException } from "@langfuse/shared/src/server";// Note: copied from official documentation: https://trpc.io/docs/server/error-handling#error-codesconst HTTP_STATUS_CODE_TO_TRPC_ERROR_CODE: Record<number, TRPC_ERROR_CODE_KEY> =@@ -27,6 +28,8 @@ const HTTP_STATUS_CODE_TO_TRPC_ERROR_CODE: Record<number, TRPC_ERROR_CODE_KEY> =const DEFAULT_ERROR_CODE: TRPC_ERROR_CODE_KEY = "INTERNAL_SERVER_ERROR";+const originalErrorKey = Symbol("trpc.originalError");+export const getTRPCErrorCodeFromHTTPStatusCode = (httpStatus: number,): TRPC_ERROR_CODE_KEY => {@@ -59,3 +62,54 @@ export const getTRPCErrorReporting = (shouldTrace: isServerErrorStatus(httpStatus),};};++/**+ * Builds the error sent to the client for a 5xx: no cause, so no stack trace+ * is exposed. The original error is kept under a non-enumerable symbol so+ * error reporting can still trace the real cause; it never reaches the+ * response.+ */+export const createScrubbedError = ({+ code,+ message,+ original,+}: {+ code: TRPC_ERROR_CODE_KEY;+ message: string;+ original: unknown;+}) => {+ const error = new TRPCError({ code, message, cause: null });+ Object.defineProperty(error, originalErrorKey, {+ value: original,+ enumerable: false,+ });+ return error;+};++const getOriginalError = (error: TRPCError): unknown =>+ (error as unknown as Record<symbol, unknown>)[originalErrorKey];++export const reportTRPCError = ({+ path,+ error,+}: {+ path: string | undefined;+ error: TRPCError;+}) => {+ const { logLevel, shouldTrace } = getTRPCErrorReporting(error);+ const message = `tRPC route failed on ${path ?? "<no-path>"}: ${error.message}`;++ if (logLevel === "error") {+ logger.error(message, error);+ } else if (logLevel === "warn") {+ logger.warn(message, error);+ } else {+ logger.info(message, error);+ }++ if (shouldTrace) {+ traceException(getOriginalError(error) ?? error);+ }++ return error;+};