🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import type { NextApiRequest, NextApiResponse } from "next";
import {
LEGACY_PUBLIC_API_METRICS_CLICKHOUSE_RESOURCE_ERROR_MESSAGE,
type RouteHandler,
withMiddlewares,
} from "@/src/features/public-api/server/withMiddlewares";
import {
clickHouseRouteForRequest,
DYNAMIC_API_ROUTE_TEMPLATES,
} from "@/src/features/public-api/server/clickHouseRequestTags";
import { readdirSync, readFileSync } from "fs";
import {
BaseError,
LangfuseNotFoundError,
UnauthorizedError,
ServiceUnavailableError,
} from "@langfuse/shared";
import {
ClickHouseResourceError,
logger,
traceException,
} from "@langfuse/shared/src/server";
import { createMocks } from "node-mocks-http";
import { z } from "zod";
import { Prisma } from "@prisma/client";
import { env } from "@/src/env.mjs";
import { sendAuthErrorJson } from "@/src/features/public-api/server/deprecations";
vi.mock("@/src/env.mjs", async (importOriginal) => {
const actual = await importOriginal<{ env: typeof env }>();
return { ...actual, env: { ...actual.env } };
});
// Mock the logger and traceException
vi.mock("@langfuse/shared/src/server", async () => ({
...(await vi.importActual("@langfuse/shared/src/server")),
logger: {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
},
traceException: vi.fn(),
}));
describe("clickHouseRouteForRequest", () => {
const request = (method: string | undefined, url: string | undefined) =>
({ method, url }) as NextApiRequest;
it("uses only the request pathname", () => {
expect(
clickHouseRouteForRequest(
request(
"GET",
"/api/public/v2/traces?projectId=project-1&secret=do-not-log",
),
),
).toBe("GET /api/public/v2/traces");
});
it("removes search params from malformed urls in the fallback path", () => {
expect(
clickHouseRouteForRequest(
request("POST", "http://[::1?secret=do-not-log#fragment"),
),
).toBe("POST http://[::1");
});
it("reports a dynamic route as its template", () => {
expect(
clickHouseRouteForRequest(
request(
"GET",
"/api/public/v2/prompts/team%2Fsupport/versions/3?label=prod",
),
),
).toBe("GET /api/public/v2/prompts/[promptName]/versions/[promptVersion]");
});
// A caller-chosen id must not survive into the route: the route tag feeds a
// metric, so every distinct id would add a series.
it.each([
"/api/public/traces/abcdefgh?traceId=x",
"/api/public/traces/api",
"/api/public/traces/traces/",
])("templates %s regardless of the id's value", (url) => {
expect(clickHouseRouteForRequest(request("GET", url))).toBe(
"GET /api/public/traces/[traceId]",
);
});
it("lists every dynamic API route that tags its ClickHouse queries", () => {
const apiDir = `${__dirname}/../../pages/api`;
const tagsQueries = (file: string) =>
file.startsWith("public/") ||
/\b(withMiddlewares|createAuthedProjectAPIRoute|clickHouseRouteForRequest)\(/.test(
readFileSync(`${apiDir}/${file}`, "utf8"),
);
const dynamicTemplates = readdirSync(apiDir, { recursive: true })
.map(String)
.filter((file) => file.endsWith(".ts") && file.includes("["))
.filter(tagsQueries)
.map(
(file) => "/api/" + file.replace(/\.ts$/, "").replace(/\/index$/, ""),
);
expect([...DYNAMIC_API_ROUTE_TEMPLATES].sort()).toEqual(
[...new Set(dynamicTemplates)].sort(),
);
});
it("reports static routes as themselves even where a template overlaps them", () => {
const apiDir = `${__dirname}/../../pages/api`;
const staticPaths = readdirSync(apiDir, { recursive: true })
.map(String)
.filter((file) => file.endsWith(".ts") && !file.includes("["))
.map(
(file) => "/api/" + file.replace(/\.ts$/, "").replace(/\/index$/, ""),
);
const misreported = staticPaths.filter(
(path) =>
clickHouseRouteForRequest(request("GET", path)) !== `GET ${path}`,
);
expect(misreported).toEqual([]);
});
it("falls back to UNKNOWN method for missing methods", () => {
expect(
clickHouseRouteForRequest(request(undefined, "/api/public/health")),
).toBe("UNKNOWN /api/public/health");
});
});
describe("withMiddlewares error handling", () => {
beforeEach(() => {
vi.clearAllMocks();
});
describe("BaseError handling", () => {
it("should handle BaseError with 4xx status code", async () => {
const error = new BaseError("BadRequest", 400, "Bad Request", false);
const handler = withMiddlewares({
POST: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(400);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Bad Request",
error: "BadRequest",
});
expect(logger.warn).toHaveBeenCalledWith(error);
expect(logger.error).not.toHaveBeenCalled();
expect(traceException).not.toHaveBeenCalled();
});
it("should handle BaseError with 5xx status code and trace exception", async () => {
const error = new BaseError(
"ServiceUnavailable",
503,
"Internal Error",
true,
);
const handler = withMiddlewares({
GET: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "GET",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(503);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Error",
error: "ServiceUnavailable",
});
// Should trace 5xx errors
expect(traceException).toHaveBeenCalledWith(error);
});
it("should set Retry-After when a ServiceUnavailableError carries one", async () => {
const handler = withMiddlewares({
POST: async () => {
throw new ServiceUnavailableError("Throttled", {
retryAfterSeconds: 2,
});
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
});
await handler(req, res);
expect(res._getStatusCode()).toBe(503);
expect(res.getHeader("Retry-After")).toBe("2");
});
});
describe("LangfuseNotFoundError handling", () => {
it("should handle LangfuseNotFoundError and log as info", async () => {
const error = new LangfuseNotFoundError("Resource not found");
const handler = withMiddlewares({
GET: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "GET",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(404);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Resource not found",
error: "LangfuseNotFoundError",
});
// Should log as info, not error
expect(logger.info).toHaveBeenCalledWith(error);
expect(logger.error).not.toHaveBeenCalled();
});
});
describe("UnauthorizedError handling", () => {
it("should handle UnauthorizedError and log as info", async () => {
const error = new UnauthorizedError("Invalid credentials");
const handler = withMiddlewares({
POST: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(401);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Invalid credentials",
error: "UnauthorizedError",
});
// Should log as info, not error
expect(logger.info).toHaveBeenCalledWith(error);
expect(logger.error).not.toHaveBeenCalled();
});
});
describe("MethodNotAllowedError handling", () => {
it("should throw MethodNotAllowedError for unsupported methods", async () => {
const handler = withMiddlewares({
GET: async () => {},
// POST is not defined
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(405);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Method not allowed",
error: "MethodNotAllowedError",
});
});
});
describe("ClickHouseResourceError handling", () => {
it("should handle ClickHouseResourceError with 422 status", async () => {
const originalError = new Error("Memory limit exceeded: maximum: 10GB");
const resourceError = new ClickHouseResourceError(
"MEMORY_LIMIT",
originalError,
);
const handler = withMiddlewares({
POST: async () => {
throw resourceError;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(422);
const jsonData = JSON.parse(res._getData());
expect(jsonData["message"]).toBeDefined();
expect(jsonData["message"]).toContain(
ClickHouseResourceError.ERROR_ADVICE_MESSAGE,
);
expect(jsonData["error"]).toBe("Request timed out");
});
it("should include tags from the error in the warn log", async () => {
const originalError = new Error("Memory limit exceeded");
const resourceError = new ClickHouseResourceError(
"MEMORY_LIMIT",
originalError,
{
tag_schema_version: "1",
surface: "publicapi",
route: "GET /api/public/test",
},
);
const handler = withMiddlewares({
GET: async () => {
throw resourceError;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "GET",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(422);
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
"ClickHouse resource limit exceeded",
expect.objectContaining({
errorType: "MEMORY_LIMIT",
tags: {
tag_schema_version: "1",
surface: "publicapi",
route: "GET /api/public/test",
},
}),
);
});
it("should handle ClickHouseResourceError with custom advice", async () => {
const originalError = new Error("Timeout exceeded");
const resourceError = new ClickHouseResourceError(
"TIMEOUT",
originalError,
);
const handler = withMiddlewares(
{
GET: async () => {
throw resourceError;
},
},
{
clickHouseResourceErrorMessage:
LEGACY_PUBLIC_API_METRICS_CLICKHOUSE_RESOURCE_ERROR_MESSAGE,
},
);
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "GET",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(422);
const jsonData = JSON.parse(res._getData());
expect(jsonData["message"]).toBe(
LEGACY_PUBLIC_API_METRICS_CLICKHOUSE_RESOURCE_ERROR_MESSAGE,
);
expect(jsonData["message"]).toContain(
"https://langfuse.com/docs/metrics/features/metrics-api",
);
});
});
describe("Prisma exception handling", () => {
it("should handle Prisma exceptions with generic 500 error", async () => {
// Create a real Prisma error
const prismaError = new Prisma.PrismaClientKnownRequestError(
"Unique constraint failed",
{ code: "P2002", clientVersion: "5.0.0" },
);
const handler = withMiddlewares({
POST: async () => {
throw prismaError;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(500);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Server Error",
error: "An unknown error occurred",
});
expect(traceException).toHaveBeenCalledWith(prismaError);
});
});
describe("Zod validation error handling", () => {
it("should handle Zod validation errors with 400 status", async () => {
const schema = z.object({
name: z.string(),
age: z.number(),
});
const handler = withMiddlewares({
POST: async () => {
// This will throw a ZodError
schema.parse({ name: "John", age: "not a number" });
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(400);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Invalid request data",
error: expect.arrayContaining([
expect.objectContaining({
code: "invalid_type",
path: ["age"],
}),
]),
});
expect(logger.warn).toHaveBeenCalledWith(expect.any(z.ZodError));
expect(logger.error).not.toHaveBeenCalled();
expect(traceException).not.toHaveBeenCalled();
});
});
describe("ServiceUnavailableError handling", () => {
it("should handle ServiceUnavailableError with 503 status", async () => {
const error = new ServiceUnavailableError(
"Storage service temporarily unavailable due to network issues",
);
const handler = withMiddlewares({
POST: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "POST",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(503);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message:
"Storage service temporarily unavailable due to network issues",
error: "ServiceUnavailableError",
});
// Should trace 5xx errors
expect(traceException).toHaveBeenCalledWith(error);
});
});
describe("Generic error handling", () => {
it("should keep handler RangeError instances on the generic 500 path", async () => {
const error = new RangeError("Invalid string length");
const handler = withMiddlewares({
GET: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "GET",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(500);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Server Error",
error: "Invalid string length",
});
expect(logger.error).toHaveBeenCalledWith(error);
expect(traceException).toHaveBeenCalledWith(error);
});
it("should handle generic Error instances with 500 status", async () => {
const error = new Error("Something went wrong");
const handler = withMiddlewares({
DELETE: async () => {
throw error;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "DELETE",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(500);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Server Error",
error: "Something went wrong",
});
expect(traceException).toHaveBeenCalledWith(error);
});
it("should handle non-Error thrown values with 500 status", async () => {
const handler = withMiddlewares({
PATCH: async () => {
throw "string error";
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "PATCH",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(500);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Server Error",
error: "An unknown error occurred",
});
});
it("should handle null/undefined errors with 500 status", async () => {
const handler = withMiddlewares({
PUT: async () => {
throw null;
},
});
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method: "PUT",
headers: {
"x-langfuse-public-key": "test-key",
},
});
await handler(req, res);
expect(res._getStatusCode()).toBe(500);
const jsonData = JSON.parse(res._getData());
expect(jsonData).toMatchObject({
message: "Internal Server Error",
error: "An unknown error occurred",
});
});
});
});
describe("withMiddlewares deprecation signal", () => {
const deprecation = {
message: "Removed on 2026-11-16. Use GET /api/public/v2/observations.",
replacement: "GET /api/public/v2/observations?fromStartTime=<from>",
docsUrl: "https://langfuse.com/docs/x",
sunsetAt: "2026-11-16",
};
const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
beforeEach(() => {
Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "US" });
});
afterEach(() => {
Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalRegion });
});
const deprecated = (fn: RouteHandler): RouteHandler =>
Object.assign(fn, { deprecation });
async function call(
handlers: Parameters<typeof withMiddlewares>[0],
method: "GET" | "POST" | "PUT",
) {
const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
method,
});
await withMiddlewares(handlers)(req, res);
return res;
}
it("sends deprecation headers and leads the body with `_deprecation`", async () => {
const res = await call(
{
GET: deprecated(async (_req, res) => {
res.status(200).json({ data: [], meta: { page: 1 } });
}),
},
"GET",
);
expect(res._getStatusCode()).toBe(200);
expect(res.getHeader("Deprecation")).toBe("@1784764800");
expect(res.getHeader("Sunset")).toBe("Mon, 16 Nov 2026 00:00:00 GMT");
expect(res.getHeader("Link")).toBe(
'</api/public/v2/observations>; rel="successor-version", <https://langfuse.com/docs/x>; rel="deprecation"',
);
const body = JSON.parse(res._getData());
expect(Object.keys(body)[0]).toBe("_deprecation");
expect(body.meta).toEqual({
deprecation: {
sunsetAt: deprecation.sunsetAt,
replacement: deprecation.replacement,
},
page: 1,
});
});
it("names the replacement in thrown 4xx errors", async () => {
const res = await call(
{
GET: deprecated(async () => {
throw new LangfuseNotFoundError("Observation not found");
}),
},
"GET",
);
expect(res._getStatusCode()).toBe(404);
const body = JSON.parse(res._getData());
expect(body._deprecation).toEqual(deprecation);
expect(body.message).toBe(
`Observation not found. Use ${deprecation.replacement} instead.`,
);
});
it("stamps a 405 for a method the deprecated route does not have", async () => {
const res = await call({ GET: deprecated(async () => {}) }, "POST");
expect(res._getStatusCode()).toBe(405);
expect(res.getHeader("Sunset")).toBe("Mon, 16 Nov 2026 00:00:00 GMT");
expect(JSON.parse(res._getData())._deprecation).toEqual(deprecation);
});
it.each([
[401, new UnauthorizedError("Invalid credentials.")],
[500, new BaseError("Internal", 500, "Database down", false)],
])(
"stamps but does not add the replacement to a %i",
async (status, error) => {
const res = await call(
{
GET: deprecated(async () => {
throw error;
}),
},
"GET",
);
expect(res._getStatusCode()).toBe(status);
const body = JSON.parse(res._getData());
expect(body._deprecation).toEqual(deprecation);
expect(body.message).toBe(error.message);
},
);
it("sends a plain 405 when the route also has live methods", async () => {
const res = await call(
{
GET: deprecated(async () => {}),
POST: async () => {},
},
"PUT",
);
expect(res._getStatusCode()).toBe(405);
expect(res.getHeader("Deprecation")).toBeUndefined();
expect(JSON.parse(res._getData())._deprecation).toBeUndefined();
});
it("stamps auth errors without the replacement hint", async () => {
const res = await call(
{
GET: deprecated(async (_req, res) => {
sendAuthErrorJson(res, 404, {
message: "Project not found or you don't have access to it",
});
}),
},
"GET",
);
expect(res._getStatusCode()).toBe(404);
const body = JSON.parse(res._getData());
expect(body._deprecation).toEqual(deprecation);
expect(body.message).toBe(
"Project not found or you don't have access to it",
);
});
it("leaves non-deprecated methods of the same route unchanged", async () => {
const res = await call(
{
GET: deprecated(async () => {}),
POST: async (_req, res) => {
res.status(201).json({ id: "1" });
},
},
"POST",
);
expect(res.getHeader("Deprecation")).toBeUndefined();
expect(JSON.parse(res._getData())).toEqual({ id: "1" });
});
it("sends no signal outside Langfuse Cloud", async () => {
Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });
const res = await call(
{
GET: deprecated(async (_req, res) => {
res.status(200).json({ data: [] });
}),
},
"GET",
);
expect(res.getHeader("Deprecation")).toBeUndefined();
expect(JSON.parse(res._getData())).toEqual({ data: [] });
});
});