Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(auth): log every shadow authorization mismatch (#18609)

Mark Salpeter10b3d8a11f21
diff --git a/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts b/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts
index 172006ad7b..c09a4dd0e3 100644
--- a/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts
+++ b/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts
@@ -143,6 +143,11 @@ describe("assertToolAuthorized", () => {
{ success: false, error: expect.any(ForbiddenError) },
{ success: true, scope: { accessLevel: "project" } },
"prompts:CUD",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
});
@@ -155,6 +160,11 @@ describe("assertToolAuthorized", () => {
{ success: true },
{ success: true, scope: { accessLevel: "project" } },
"prompts:read",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
});
});
diff --git a/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts
index 1e8784d1f9..13d91974fe 100644
--- a/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts
+++ b/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts
@@ -1,7 +1,10 @@
import { readFileSync } from "fs";
import { fileURLToPath } from "url";
+import { trace } from "@opentelemetry/api";
import { type NextApiRequest } from "next";
+import { type RouteKind } from "next/dist/server/route-kind";
+import { setRequestMeta } from "next/dist/server/request-meta";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
@@ -40,6 +43,7 @@ vi.mock("@/src/features/public-api/server/verifyProjectApiKeyAuth", () => ({
}));
vi.mock("@/src/features/public-api/server/enforceAuth", () => ({
+ __dangerouslySkipAuthz: "__dangerouslySkipAuthz",
enforceAuth: mockEnforceAuth,
}));
@@ -51,7 +55,8 @@ vi.mock(
}),
);
-import { shadowAuth } from "@/src/features/public-api/server";
+import { shadowAuth, shadowAuthorize } from "@/src/features/public-api/server";
+import { type AuthorizationContext } from "@/src/features/auth/policy/types";
const mappedFields = {
orgId: "org_1",
@@ -149,6 +154,7 @@ describe("org-family dispatch (allowedAccessLevels ['organization'])", () => {
{ success: false, error: expect.any(ForbiddenError) },
{ success: true, scope: orgScope },
"projects:read",
+ expect.objectContaining({ resource: "GET unknown" }),
);
});
});
@@ -360,6 +366,7 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {
newFailure,
expect.objectContaining({ success: false }),
"traces:read",
+ expect.objectContaining({ resource: "GET unknown" }),
);
} else {
expect(mockEnforceAuth).not.toHaveBeenCalled();
@@ -426,6 +433,7 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {
{ success: false, error: expect.any(ForbiddenError) },
{ success: true, scope: legacyScope.scope },
"traces:read",
+ expect.objectContaining({ resource: "GET unknown" }),
);
});
@@ -495,6 +503,199 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {
});
});
+describe("shadow parity metadata", () => {
+ const scope = {
+ ...mappedFields,
+ projectId: "prj_1",
+ accessLevel: "project" as const,
+ };
+ const authContext = (): AuthorizationContext => ({
+ principal: {
+ kind: "apiKey",
+ apiKeyId: "key_1",
+ userId: null,
+ isInAppAgentKey: false,
+ publicKey: "pk-lf-1",
+ scope: "PROJECT",
+ presentation: "privateKey",
+ organizations: [],
+ boundResource: { orgId: "org_1", projectId: "prj_1" },
+ },
+ policies: [],
+ });
+ const request = (route?: string) => {
+ const req = {
+ headers: { authorization: "Basic secret-credentials" },
+ method: "GET",
+ url: "/api/public/prompts/private-prompt?secret=private-query",
+ } as NextApiRequest;
+ if (route) {
+ setRequestMeta(req, {
+ match: {
+ params: undefined,
+ definition: {
+ kind: "PAGES_API" as RouteKind.PAGES_API,
+ pathname: route,
+ page: route,
+ bundlePath: `pages${route}`,
+ filename: "handler.js",
+ },
+ },
+ });
+ }
+ return req;
+ };
+ const call = (req: NextApiRequest) =>
+ shadowAuth({
+ req,
+ action: "prompts:read",
+ allowedAccessLevels: ["project"],
+ });
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ env.API_AUTH_MIGRATION = "shadow";
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it.each([0, 1])(
+ "uses the matched route template with trace flags %s",
+ async (traceFlags) => {
+ vi.spyOn(trace, "getActiveSpan").mockReturnValue(
+ trace.wrapSpanContext({
+ traceId: "12345678901234567890123456789012",
+ spanId: "1234567890123456",
+ traceFlags,
+ }),
+ );
+ mockLegacyProjectAuth.mockResolvedValue({ scope });
+ mockEnforceAuth.mockResolvedValue({
+ success: false,
+ error: new ForbiddenError("new denial"),
+ });
+
+ await call(request("/api/public/prompts/[promptName]"));
+
+ expect(mockShadowAuthDiff).toHaveBeenCalledWith(
+ expect.objectContaining({ success: false }),
+ { success: true, scope },
+ "prompts:read",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET /api/public/prompts/[promptName]",
+ },
+ );
+ },
+ );
+
+ it("retains the new allow's safe key identity on every repeated mismatch", async () => {
+ mockLegacyProjectAuth.mockRejectedValue({ status: 403, message: "legacy" });
+ mockEnforceAuth.mockResolvedValue({
+ success: true,
+ scope,
+ ctx: authContext(),
+ });
+ const req = request("/api/public/prompts/[promptName]");
+
+ await call(req);
+ await call(req);
+
+ expect(mockShadowAuthDiff).toHaveBeenCalledTimes(2);
+ for (const args of mockShadowAuthDiff.mock.calls) {
+ expect(args[3]).toEqual({
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET /api/public/prompts/[promptName]",
+ });
+ }
+ });
+
+ it("carries the request resource into every asynchronous per-item check", async () => {
+ mockLegacyProjectAuth.mockResolvedValue({ scope });
+ mockEnforceAuth.mockResolvedValue({
+ success: true,
+ scope,
+ ctx: authContext(),
+ });
+ const req = request("/api/public/mcp");
+ const result = await call(req);
+ expect(result.success).toBe(true);
+ if (!result.success) throw new Error("Expected successful legacy auth");
+ setRequestMeta(req, {});
+ mockShadowAuthDiff.mockClear();
+
+ await Promise.all(
+ [1, 2, 3].map(async () => {
+ await Promise.resolve();
+ shadowAuthorize({
+ ctx: result.ctx,
+ action: "prompts:CUD",
+ resource: { projectId: "prj_1" },
+ legacyDecision: { success: true, scope },
+ });
+ }),
+ );
+
+ expect(mockShadowAuthDiff).toHaveBeenCalledTimes(3);
+ for (const args of mockShadowAuthDiff.mock.calls) {
+ expect(args[0]).toMatchObject({ success: false });
+ expect(args[3]).toEqual({
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET /api/public/mcp",
+ });
+ }
+ });
+
+ it("isolates request resources when authentication reuses the same context", async () => {
+ mockLegacyProjectAuth.mockResolvedValue({ scope });
+ mockEnforceAuth.mockResolvedValue({
+ success: true,
+ scope,
+ ctx: authContext(),
+ });
+ const first = await call(request("/api/public/mcp"));
+ const second = await call(request("/api/public/ingestion"));
+ if (!first.success || !second.success) {
+ throw new Error("Expected successful legacy auth");
+ }
+ mockShadowAuthDiff.mockClear();
+
+ for (const result of [first, second]) {
+ shadowAuthorize({
+ ctx: result.ctx,
+ action: "prompts:CUD",
+ resource: { projectId: "prj_1" },
+ legacyDecision: { success: true, scope },
+ });
+ }
+
+ expect(
+ mockShadowAuthDiff.mock.calls.map((args) => args[3].resource),
+ ).toEqual(["GET /api/public/mcp", "GET /api/public/ingestion"]);
+ });
+
+ it("never falls back to a concrete URL when route metadata is unavailable", async () => {
+ mockLegacyProjectAuth.mockResolvedValue({ scope });
+ mockEnforceAuth.mockResolvedValue({
+ success: false,
+ error: new ForbiddenError("new denial"),
+ });
+
+ await call(request());
+
+ expect(mockShadowAuthDiff.mock.calls[0][3]).toEqual({
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET unknown",
+ });
+ });
+});
+
describe("the migration-mode gate reads a declared env var", () => {
const source = (relativeToSrc: string) =>
readFileSync(
diff --git a/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts
index 77c625e2a9..6db1482baf 100644
--- a/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts
+++ b/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts
@@ -1,4 +1,6 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { logger } from "@langfuse/shared/src/server";
import {
__test,
@@ -29,6 +31,100 @@ const legacyDeny = {
} as LegacyDecision;
describe("shadowAuthDiff — the ship-gate signal", () => {
+ afterEach(() => vi.restoreAllMocks());
+
+ it("adds the route resource to metrics without duplicating it on the span", () => {
+ vi.spyOn(logger, "info").mockImplementation(() => logger);
+ const { calls, telemetry } = capture();
+ const setAttribute = vi.fn();
+ telemetry.span = () => ({ setAttribute });
+
+ shadowAuthDiff(
+ deny,
+ legacyAllow,
+ "prompts:read",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET /api/public/prompts/[promptName]",
+ },
+ telemetry,
+ );
+
+ expect(Object.fromEntries(setAttribute.mock.calls)).toEqual({
+ "langfuse.authz.parity.action": "prompts:read",
+ "langfuse.authz.parity.access_level": "project",
+ "langfuse.authz.parity.result": "new_denies",
+ "langfuse.authz.parity.legacy_code": 200,
+ "langfuse.authz.parity.new_code": 403,
+ });
+ expect(calls[0].tags).toEqual({
+ action: "prompts:read",
+ resource: "GET /api/public/prompts/[promptName]",
+ access_level: "project",
+ result: "new_denies",
+ legacy_code: 200,
+ new_code: 403,
+ });
+ });
+
+ it.each([
+ [deny, legacyAllow, "new_denies", 200, 403],
+ [allow, legacyDeny, "new_allows", 403, 200],
+ ] as const)(
+ "logs every mismatch with codes %s / %s without a recording span",
+ (newDecision, legacyDecision, result, legacyCode, newCode) => {
+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
+ const { telemetry } = capture();
+ const metadata = {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "GET /api/public/prompts/[promptName]",
+ secretKey: "sk-lf-never-log",
+ };
+ for (let i = 0; i < 3; i++) {
+ shadowAuthDiff(
+ newDecision,
+ legacyDecision,
+ "prompts:read",
+ metadata,
+ telemetry,
+ );
+ }
+ expect(log).toHaveBeenCalledTimes(3);
+ expect(log).toHaveBeenCalledWith("Public API authorization mismatch", {
+ event: "authz_parity",
+ action: "prompts:read",
+ access_level: result === "new_denies" ? "project" : "unknown",
+ result,
+ legacy_code: legacyCode,
+ new_code: newCode,
+ api_key_id: "key_1",
+ public_key: "pk-lf-1",
+ resource: "GET /api/public/prompts/[promptName]",
+ });
+ },
+ );
+
+ it("does not log matches or net-new gates", () => {
+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
+ const { telemetry } = capture();
+ for (const [newDecision, legacyDecision] of [
+ [allow, legacyAllow],
+ [deny, legacyDeny],
+ [deny, { absent: true }],
+ ] as const) {
+ shadowAuthDiff(
+ newDecision,
+ legacyDecision,
+ "traces:read",
+ { resource: "unknown" },
+ telemetry,
+ );
+ }
+ expect(log).not.toHaveBeenCalled();
+ });
+
it.each([
["both allow", allow, legacyAllow, "match"],
["both deny", deny, legacyDeny, "match"],
@@ -38,7 +134,13 @@ describe("shadowAuthDiff — the ship-gate signal", () => {
] as const)("%s", (_name, newVerdict, legacyVerdict, result) => {
const { calls, telemetry } = capture();
expect(
- shadowAuthDiff(newVerdict, legacyVerdict, "traces:read", telemetry),
+ shadowAuthDiff(
+ newVerdict,
+ legacyVerdict,
+ "traces:read",
+ { resource: "unknown" },
+ telemetry,
+ ),
).toBe(result);
expect(calls[0].tags.result).toBe(result);
});
diff --git a/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts
index 78ef6aa29e..4c8274c371 100644
--- a/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts
+++ b/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts
@@ -136,6 +136,15 @@ describe("shadowAuthorize", () => {
});
});
+ it.each(["legacy", "enforce"])(
+ "does not compare per-item decisions in %s even with an auth context",
+ (mode) => {
+ env.API_AUTH_MIGRATION = mode;
+ shadowAuthorize(params("prompts:CUD", authContext([allowPrompts])));
+ expect(shadowAuthDiff).not.toHaveBeenCalled();
+ },
+ );
+
describe("shadow mode", () => {
beforeEach(() => {
env.API_AUTH_MIGRATION = "shadow";
@@ -153,6 +162,11 @@ describe("shadowAuthorize", () => {
{ success: false, error: expect.any(ForbiddenError) },
{ success: true, scope: { accessLevel: "project" } },
"prompts:CUD",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
});
@@ -175,6 +189,11 @@ describe("shadowAuthorize", () => {
: { success: false, error: expect.any(ForbiddenError) },
legacyDecision,
action,
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
},
);
@@ -185,6 +204,11 @@ describe("shadowAuthorize", () => {
{ success: true },
{ success: true, scope: { accessLevel: "project" } },
"prompts:read",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
});
});
@@ -243,6 +267,11 @@ describe("protected-label legacy comparisons", () => {
{ success: true },
{ success: false, error: expect.any(ForbiddenError) },
"promptProtectedLabels:CUD",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
} else {
expect(shadowAuthDiff).not.toHaveBeenCalled();
@@ -269,6 +298,11 @@ describe("protected-label legacy comparisons", () => {
{ success: true },
{ success: true, scope: { accessLevel: "project" } },
"promptProtectedLabels:CUD",
+ {
+ apiKeyId: "key_1",
+ publicKey: "pk-lf-1",
+ resource: "unknown",
+ },
);
});
diff --git a/web/src/features/public-api/server/shadowAuth.ts b/web/src/features/public-api/server/shadowAuth.ts
index 241c78e132..f7414837d1 100644
--- a/web/src/features/public-api/server/shadowAuth.ts
+++ b/web/src/features/public-api/server/shadowAuth.ts
@@ -1,4 +1,5 @@
import { type NextApiRequest } from "next";
+import { getRequestMeta } from "next/dist/server/request-meta";
import { ApiError, isBaseError, type BaseError } from "@langfuse/shared";
import {
@@ -45,6 +46,9 @@ import {
} from "@/src/features/auth/policy/types";
import { isPrismaException } from "@/src/utils/exceptions";
+/** shadowResources retains request route templates for asynchronous per-item checks. */
+const shadowResources = new WeakMap<AuthorizationContext, string>();
+
/** shadowAuth authorizes a public-API request under the active migration mode, returning the scope or the error to render. */
export async function shadowAuth(
params: ShadowAuthParams,
@@ -105,14 +109,44 @@ function genericInfrastructureError(httpCode: number): ErrorResult<BaseError> {
async function legacyWithShadow(
params: ShadowAuthParams,
): Promise<ShadowAuthResult> {
+ const resource = requestResource(params.req);
const legacyAuth = await runLegacyAuth(params);
const newAuth = await runNewAuth(params);
- shadowAuthDiff(newAuth, legacyAuth, params.action);
+ shadowAuthDiff(newAuth, legacyAuth, params.action, {
+ ...getScopeIdentity(newAuth, legacyAuth),
+ resource,
+ });
const result = legacyResult(legacyAuth);
- if (result.success && newAuth.success) return { ...result, ctx: newAuth.ctx };
+ if (result.success && newAuth.success) {
+ return { ...result, ctx: withShadowResource(newAuth.ctx, resource) };
+ }
return result;
}
+function getScopeIdentity(
+ newAuth: EnforceAuthResult,
+ legacyAuth: LegacyDecision,
+) {
+ const successfulAuth = newAuth.success ? newAuth : legacyAuth;
+ const scope = successfulAuth.success ? successfulAuth.scope : undefined;
+ return { apiKeyId: scope?.apiKeyId, publicKey: scope?.publicKey };
+}
+
+function withShadowResource(
+ ctx: AuthorizationContext,
+ resource: string,
+): AuthorizationContext {
+ const shadowCtx = { ...ctx };
+ shadowResources.set(shadowCtx, resource);
+ return shadowCtx;
+}
+
+/** requestResource reads Next's matched route independently of recording spans. */
+function requestResource(req: NextApiRequest): string {
+ const route = getRequestMeta(req, "match")?.definition.pathname;
+ return `${req.method ?? "UNKNOWN"} ${route ?? "unknown"}`;
+}
+
/** legacyOnly authorizes solely with the legacy verify. */
async function legacyOnly(params: ShadowAuthParams): Promise<ShadowAuthResult> {
return legacyResult(await runLegacyAuth(params));
@@ -221,12 +255,36 @@ function diffOrGate(
decision: Decision,
): Decision {
if (env.API_AUTH_MIGRATION === "shadow") {
- shadowAuthDiff(decision, params.legacyDecision, params.action);
+ shadowAuthDiff(
+ decision,
+ params.legacyDecision,
+ params.action,
+ getAuthMetadata(params.ctx),
+ );
return params.legacyDecision;
}
return decision;
}
+function getAuthMetadata(ctx: AuthorizationContext | undefined) {
+ return { ...getPrincipalIdentity(ctx), resource: getResource(ctx) };
+}
+
+function getPrincipalIdentity(ctx: AuthorizationContext | undefined) {
+ const principal = ctx?.principal;
+ if (principal?.kind === "apiKey") {
+ return { apiKeyId: principal.apiKeyId, publicKey: principal.publicKey };
+ }
+ if (principal?.kind === "admin") {
+ return { apiKeyId: "ADMIN_API_KEY", publicKey: "ADMIN_API_KEY" };
+ }
+ return { apiKeyId: undefined, publicKey: undefined };
+}
+
+function getResource(ctx: AuthorizationContext | undefined): string {
+ return (ctx && shadowResources.get(ctx)) ?? "unknown";
+}
+
/** toResourceId tags a per-item resource for the PDP. */
function toResourceId(resource: Resource): ResourceId {
return "projectId" in resource
diff --git a/web/src/features/public-api/server/shadowAuthDiff.ts b/web/src/features/public-api/server/shadowAuthDiff.ts
index 7ecaaf5d9c..1a695b1ce1 100644
--- a/web/src/features/public-api/server/shadowAuthDiff.ts
+++ b/web/src/features/public-api/server/shadowAuthDiff.ts
@@ -1,5 +1,6 @@
import {
getCurrentSpan,
+ logger,
recordIncrement,
type ApiAccessLevel,
} from "@langfuse/shared/src/server";
@@ -13,27 +14,57 @@ const defaultTelemetry: Telemetry = {
span: () => getCurrentSpan(),
};
-/** shadowAuthDiff classifies the new pipeline's decision against legacy's, emits `langfuse.authz.parity`, and returns the result. */
+/** shadowAuthDiff records parity and logs every mismatch independently of trace sampling. */
export function shadowAuthDiff(
newDecision: NewDecision,
legacyDecision: LegacyDecision,
action: string,
+ metadata: AuthDiffMetadata,
telemetry: Telemetry = defaultTelemetry,
): ParityResult {
const n = newVerdict(newDecision);
const l = legacyVerdict(legacyDecision);
const result = classify(l.verdict, n.verdict);
- telemetry.increment(parityStat, {
+ const tags = {
action,
access_level: l.accessLevel,
result,
legacy_code: l.code,
new_code: n.code,
- });
- telemetry.span()?.setAttribute(`${parityStat}.result`, result);
+ };
+ logMismatch(result, tags, metadata);
+ telemetry.increment(parityStat, { ...tags, resource: metadata.resource });
+ annotateSpan(telemetry.span(), tags);
return result;
}
+/** logMismatch emits only safe identity and route fields for disagreements. */
+function logMismatch(
+ result: ParityResult,
+ tags: Record<string, string | number>,
+ metadata: AuthDiffMetadata,
+): void {
+ if (result !== "new_allows" && result !== "new_denies") return;
+ logger.info("Public API authorization mismatch", {
+ event: "authz_parity",
+ ...tags,
+ api_key_id: metadata.apiKeyId,
+ public_key: metadata.publicKey,
+ resource: metadata.resource,
+ });
+}
+
+/** annotateSpan adds decision tags under the parity namespace. */
+function annotateSpan(
+ span: ReturnType<Telemetry["span"]>,
+ tags: Record<string, string | number>,
+): void {
+ if (!span) return;
+ for (const [key, value] of Object.entries(tags)) {
+ span.setAttribute(`${parityStat}.${key}`, value);
+ }
+}
+
/** newVerdict reads the new decision: success allows (200), else denies with its http code. */
function newVerdict(decision: NewDecision): { verdict: Verdict; code: number } {
return decision.success
@@ -68,6 +99,13 @@ function classify(legacyVerdict: Verdict, newVerdict: Verdict): ParityResult {
/** Verdict is one path's decision at one enforcement point; legacy is `absent` only where it runs no gate. */
type Verdict = "allow" | "deny" | "absent";
+/** AuthDiffMetadata carries safe key identity and the resolved HTTP route resource. */
+type AuthDiffMetadata = {
+ apiKeyId?: string;
+ publicKey?: string;
+ resource: string;
+};
+
/** ParityResult is the signal: agreement, which path is stricter, or net-new enforcement legacy never gated. */
export type ParityResult = "match" | "new_denies" | "new_allows" | "net_new";