Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: trace the real cause of tRPC 5xx errors and nest Datadog log trace ids (#18546)

Niklas Semmler911c91ad7f4b
diff --git a/packages/shared/src/server/logger.test.ts b/packages/shared/src/server/logger.test.ts
index 09f285dde5..235f6c74fa 100644
--- a/packages/shared/src/server/logger.test.ts
+++ b/packages/shared/src/server/logger.test.ts
@@ -8,6 +8,9 @@ vi.mock("../env", () => ({
},
}));
+const getCurrentSpan = vi.hoisted(() => vi.fn());
+vi.mock("./instrumentation", () => ({ getCurrentSpan }));
+
import { logger } from "./logger";
const jsonLine = (meta: Record<string, unknown>) => {
@@ -65,3 +68,25 @@ describe("logger JSON format", () => {
});
});
});
+
+describe("logger trace correlation", () => {
+ it("nests the Datadog ids under dd and keeps the hex OTel ids", () => {
+ getCurrentSpan.mockReturnValueOnce({
+ spanContext: () => ({
+ traceId: "617222771252365135e4848d739614a8",
+ spanId: "00f067aa0ba902b7",
+ }),
+ });
+
+ const line = jsonLine({});
+
+ expect(line.dd).toEqual({
+ trace_id: "3883374521764680872",
+ span_id: "67667974448284343",
+ });
+ expect(line).not.toHaveProperty(["dd.trace_id"]);
+ expect(line).not.toHaveProperty(["dd.span_id"]);
+ expect(line.trace_id).toBe("617222771252365135e4848d739614a8");
+ expect(line.span_id).toBe("00f067aa0ba902b7");
+ });
+});
diff --git a/packages/shared/src/server/logger.ts b/packages/shared/src/server/logger.ts
index 1979b274f0..2138781d0e 100644
--- a/packages/shared/src/server/logger.ts
+++ b/packages/shared/src/server/logger.ts
@@ -9,8 +9,10 @@ const tracingFormat = function () {
if (span) {
const { spanId, traceId } = span.spanContext();
const traceIdEnd = traceId.slice(traceId.length / 2);
- info["dd.trace_id"] = BigInt(`0x${traceIdEnd}`).toString();
- info["dd.span_id"] = BigInt(`0x${spanId}`).toString();
+ info.dd = {
+ trace_id: BigInt(`0x${traceIdEnd}`).toString(),
+ span_id: BigInt(`0x${spanId}`).toString(),
+ };
info["trace_id"] = traceId;
info["span_id"] = spanId;
}
diff --git a/web/src/__tests__/server/trpc-error-formatting.servertest.ts b/web/src/__tests__/server/trpc-error-formatting.servertest.ts
index 5d013095a5..77e933314f 100644
--- a/web/src/__tests__/server/trpc-error-formatting.servertest.ts
+++ b/web/src/__tests__/server/trpc-error-formatting.servertest.ts
@@ -6,12 +6,20 @@ vi.mock("@langfuse/shared/src/server", async () => ({
error: vi.fn(),
debug: vi.fn(),
},
+ traceException: vi.fn(),
}));
import type { Session } from "next-auth";
import { TRPCError } from "@trpc/server";
import * as z from "zod";
-import { ClickHouseResourceError, logger } from "@langfuse/shared/src/server";
+import {
+ ClickHouseResourceError,
+ logger,
+ traceException,
+} from "@langfuse/shared/src/server";
+import { Prisma } from "@langfuse/shared/src/db";
+import { fetchRequestHandler } from "@trpc/server/adapters/fetch";
+import { reportTRPCError } from "@/src/server/utils/trpc-utils";
import {
createInnerTRPCContext,
createTRPCRouter,
@@ -130,4 +138,64 @@ describe("tRPC error formatting", () => {
expect(formattedWithStack.data["stack"]).toBe("dev stack");
});
+
+ describe("over HTTP", () => {
+ const fetchFailing = (thrown: unknown) => {
+ const router = createTRPCRouter({
+ failing: protectedProcedureWithoutTracing.query(() => {
+ throw thrown;
+ }),
+ });
+ return fetchRequestHandler({
+ endpoint: "/api/trpc",
+ req: new Request("http://localhost/api/trpc/failing"),
+ router,
+ createContext: () =>
+ createInnerTRPCContext({
+ session: { user: { id: "user-1" } } as Session,
+ headers: {},
+ }),
+ onError: reportTRPCError,
+ });
+ };
+
+ it.each([
+ [
+ "Prisma",
+ new Prisma.PrismaClientKnownRequestError(
+ "Unique constraint failed on the fields: (`id`)",
+ { code: "P2002", clientVersion: "test" },
+ ),
+ ],
+ [
+ "uncaused TRPCError",
+ new TRPCError({
+ code: "INTERNAL_SERVER_ERROR",
+ message: "connection refused",
+ }),
+ ],
+ ])(
+ "traces the %s cause on the span but returns a generic 5xx",
+ async (_, original) => {
+ const res = await fetchFailing(original);
+ const body = await res.text();
+
+ expect(res.status).toBe(500);
+ expect(body).toContain("Internal error. ");
+ expect(body).not.toContain(original.message);
+ expect(traceException).toHaveBeenCalledTimes(1);
+ expect(vi.mocked(traceException).mock.calls[0]?.[0]).toBe(original);
+ },
+ );
+
+ it("keeps the message of a 4xx error and does not trace it", async () => {
+ const res = await fetchFailing(
+ new TRPCError({ code: "BAD_REQUEST", message: "Invalid filter" }),
+ );
+
+ expect(res.status).toBe(400);
+ expect(await res.text()).toContain("Invalid filter");
+ expect(traceException).not.toHaveBeenCalled();
+ });
+ });
});
diff --git a/web/src/pages/api/trpc/[trpc].ts b/web/src/pages/api/trpc/[trpc].ts
index 573a253926..d0cafa3772 100644
--- a/web/src/pages/api/trpc/[trpc].ts
+++ b/web/src/pages/api/trpc/[trpc].ts
@@ -2,8 +2,7 @@ import { createNextApiHandler } from "@trpc/server/adapters/next";
import { createTRPCContext } from "@/src/server/api/trpc";
import { appRouter } from "@/src/server/api/root";
import { env } from "@/src/env.mjs";
-import { logger, traceException } from "@langfuse/shared/src/server";
-import { getTRPCErrorReporting } from "@/src/server/utils/trpc-utils";
+import { reportTRPCError } from "@/src/server/utils/trpc-utils";
export const config = {
maxDuration: 240,
@@ -25,24 +24,7 @@ export default createNextApiHandler({
// option), but it only widens the accepted method for queries (read-only);
// mutations remain POST-only, so the GET-mutation protection is unchanged.
allowMethodOverride: true,
- onError: ({ path, error }) => {
- const { logLevel, shouldTrace } = getTRPCErrorReporting(error);
- const message = `tRPC route failed on ${path ?? "<no-path>"}: ${error.message}`;
-
- if (logLevel === "error") {
- logger.error(message, error);
- } else if (logLevel === "warn") {
- logger.warn(message, error);
- } else {
- logger.info(message, error);
- }
-
- if (shouldTrace) {
- traceException(error);
- }
-
- return error;
- },
+ onError: reportTRPCError,
responseMeta() {
return {
headers: {
diff --git a/web/src/server/api/trpc.ts b/web/src/server/api/trpc.ts
index ee0fe57334..a57445fcc4 100644
--- a/web/src/server/api/trpc.ts
+++ b/web/src/server/api/trpc.ts
@@ -23,7 +23,10 @@ import { prisma, Role } from "@langfuse/shared/src/db";
import * as z from "zod";
import * as opentelemetry from "@opentelemetry/api";
import { type IncomingHttpHeaders } from "node:http";
-import { getTRPCErrorCodeFromHTTPStatusCode } from "@/src/server/utils/trpc-utils";
+import {
+ createScrubbedError,
+ getTRPCErrorCodeFromHTTPStatusCode,
+} from "@/src/server/utils/trpc-utils";
import { sendAdminAccessWebhook } from "@/src/server/adminAccessWebhook";
type CreateContextOptions = {
@@ -217,13 +220,19 @@ const withErrorHandling = t.middleware(async ({ ctx, next }) => {
: "Please check error logs in your self-hosted deployment.";
logErrorByStatus({ errorCode: code, httpStatus, error: res.error });
- res.error = new TRPCError({
- code,
- cause: null, // do not expose stack traces
- message: isSafeToExpose
- ? res.error.message
- : "Internal error. " + errorMessage,
- });
+ if (isSafeToExpose) {
+ res.error = new TRPCError({
+ code,
+ cause: null, // do not expose stack traces
+ message: res.error.message,
+ });
+ } else {
+ res.error = createScrubbedError({
+ code,
+ message: "Internal error. " + errorMessage,
+ original: res.error.cause ?? res.error,
+ });
+ }
}
}
diff --git a/web/src/server/utils/trpc-utils.ts b/web/src/server/utils/trpc-utils.ts
index 7f3c52c060..91f0b6b8d7 100644
--- a/web/src/server/utils/trpc-utils.ts
+++ b/web/src/server/utils/trpc-utils.ts
@@ -1,5 +1,6 @@
-import type { TRPCError, TRPC_ERROR_CODE_KEY } from "@trpc/server";
+import { TRPCError, type TRPC_ERROR_CODE_KEY } from "@trpc/server";
import { getHTTPStatusCodeFromError } from "@trpc/server/http";
+import { logger, traceException } from "@langfuse/shared/src/server";
// Note: copied from official documentation: https://trpc.io/docs/server/error-handling#error-codes
const HTTP_STATUS_CODE_TO_TRPC_ERROR_CODE: Record<number, TRPC_ERROR_CODE_KEY> =
@@ -27,6 +28,8 @@ const HTTP_STATUS_CODE_TO_TRPC_ERROR_CODE: Record<number, TRPC_ERROR_CODE_KEY> =
const DEFAULT_ERROR_CODE: TRPC_ERROR_CODE_KEY = "INTERNAL_SERVER_ERROR";
+const originalErrorKey = Symbol("trpc.originalError");
+
export const getTRPCErrorCodeFromHTTPStatusCode = (
httpStatus: number,
): TRPC_ERROR_CODE_KEY => {
@@ -59,3 +62,54 @@ export const getTRPCErrorReporting = (
shouldTrace: isServerErrorStatus(httpStatus),
};
};
+
+/**
+ * Builds the error sent to the client for a 5xx: no cause, so no stack trace
+ * is exposed. The original error is kept under a non-enumerable symbol so
+ * error reporting can still trace the real cause; it never reaches the
+ * response.
+ */
+export const createScrubbedError = ({
+ code,
+ message,
+ original,
+}: {
+ code: TRPC_ERROR_CODE_KEY;
+ message: string;
+ original: unknown;
+}) => {
+ const error = new TRPCError({ code, message, cause: null });
+ Object.defineProperty(error, originalErrorKey, {
+ value: original,
+ enumerable: false,
+ });
+ return error;
+};
+
+const getOriginalError = (error: TRPCError): unknown =>
+ (error as unknown as Record<symbol, unknown>)[originalErrorKey];
+
+export const reportTRPCError = ({
+ path,
+ error,
+}: {
+ path: string | undefined;
+ error: TRPCError;
+}) => {
+ const { logLevel, shouldTrace } = getTRPCErrorReporting(error);
+ const message = `tRPC route failed on ${path ?? "<no-path>"}: ${error.message}`;
+
+ if (logLevel === "error") {
+ logger.error(message, error);
+ } else if (logLevel === "warn") {
+ logger.warn(message, error);
+ } else {
+ logger.info(message, error);
+ }
+
+ if (shouldTrace) {
+ traceException(getOriginalError(error) ?? error);
+ }
+
+ return error;
+};