🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(auth): log every shadow authorization mismatch (#18609)
Mark Salpeter10b3d8a11f21
diff --git a/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts b/web/src/__tests__/server/unit/assertToolAuthorized.servertest.tsindex 172006ad7b..c09a4dd0e3 100644--- a/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts+++ b/web/src/__tests__/server/unit/assertToolAuthorized.servertest.ts@@ -143,6 +143,11 @@ describe("assertToolAuthorized", () => {{ success: false, error: expect.any(ForbiddenError) },{ success: true, scope: { accessLevel: "project" } },"prompts:CUD",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);});@@ -155,6 +160,11 @@ describe("assertToolAuthorized", () => {{ success: true },{ success: true, scope: { accessLevel: "project" } },"prompts:read",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);});});diff --git a/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.tsindex 1e8784d1f9..13d91974fe 100644--- a/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts+++ b/web/src/__tests__/server/unit/public-api/shadowAuth.servertest.ts@@ -1,7 +1,10 @@import { readFileSync } from "fs";import { fileURLToPath } from "url";+import { trace } from "@opentelemetry/api";import { type NextApiRequest } from "next";+import { type RouteKind } from "next/dist/server/route-kind";+import { setRequestMeta } from "next/dist/server/request-meta";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import {@@ -40,6 +43,7 @@ vi.mock("@/src/features/public-api/server/verifyProjectApiKeyAuth", () => ({}));vi.mock("@/src/features/public-api/server/enforceAuth", () => ({+ __dangerouslySkipAuthz: "__dangerouslySkipAuthz",enforceAuth: mockEnforceAuth,}));@@ -51,7 +55,8 @@ vi.mock(}),);-import { shadowAuth } from "@/src/features/public-api/server";+import { shadowAuth, shadowAuthorize } from "@/src/features/public-api/server";+import { type AuthorizationContext } from "@/src/features/auth/policy/types";const mappedFields = {orgId: "org_1",@@ -149,6 +154,7 @@ describe("org-family dispatch (allowedAccessLevels ['organization'])", () => {{ success: false, error: expect.any(ForbiddenError) },{ success: true, scope: orgScope },"projects:read",+ expect.objectContaining({ resource: "GET unknown" }),);});});@@ -360,6 +366,7 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {newFailure,expect.objectContaining({ success: false }),"traces:read",+ expect.objectContaining({ resource: "GET unknown" }),);} else {expect(mockEnforceAuth).not.toHaveBeenCalled();@@ -426,6 +433,7 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {{ success: false, error: expect.any(ForbiddenError) },{ success: true, scope: legacyScope.scope },"traces:read",+ expect.objectContaining({ resource: "GET unknown" }),);});@@ -495,6 +503,199 @@ describe("project-family dispatch (allowedAccessLevels ['project'])", () => {});});+describe("shadow parity metadata", () => {+ const scope = {+ ...mappedFields,+ projectId: "prj_1",+ accessLevel: "project" as const,+ };+ const authContext = (): AuthorizationContext => ({+ principal: {+ kind: "apiKey",+ apiKeyId: "key_1",+ userId: null,+ isInAppAgentKey: false,+ publicKey: "pk-lf-1",+ scope: "PROJECT",+ presentation: "privateKey",+ organizations: [],+ boundResource: { orgId: "org_1", projectId: "prj_1" },+ },+ policies: [],+ });+ const request = (route?: string) => {+ const req = {+ headers: { authorization: "Basic secret-credentials" },+ method: "GET",+ url: "/api/public/prompts/private-prompt?secret=private-query",+ } as NextApiRequest;+ if (route) {+ setRequestMeta(req, {+ match: {+ params: undefined,+ definition: {+ kind: "PAGES_API" as RouteKind.PAGES_API,+ pathname: route,+ page: route,+ bundlePath: `pages${route}`,+ filename: "handler.js",+ },+ },+ });+ }+ return req;+ };+ const call = (req: NextApiRequest) =>+ shadowAuth({+ req,+ action: "prompts:read",+ allowedAccessLevels: ["project"],+ });++ beforeEach(() => {+ vi.clearAllMocks();+ env.API_AUTH_MIGRATION = "shadow";+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it.each([0, 1])(+ "uses the matched route template with trace flags %s",+ async (traceFlags) => {+ vi.spyOn(trace, "getActiveSpan").mockReturnValue(+ trace.wrapSpanContext({+ traceId: "12345678901234567890123456789012",+ spanId: "1234567890123456",+ traceFlags,+ }),+ );+ mockLegacyProjectAuth.mockResolvedValue({ scope });+ mockEnforceAuth.mockResolvedValue({+ success: false,+ error: new ForbiddenError("new denial"),+ });++ await call(request("/api/public/prompts/[promptName]"));++ expect(mockShadowAuthDiff).toHaveBeenCalledWith(+ expect.objectContaining({ success: false }),+ { success: true, scope },+ "prompts:read",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET /api/public/prompts/[promptName]",+ },+ );+ },+ );++ it("retains the new allow's safe key identity on every repeated mismatch", async () => {+ mockLegacyProjectAuth.mockRejectedValue({ status: 403, message: "legacy" });+ mockEnforceAuth.mockResolvedValue({+ success: true,+ scope,+ ctx: authContext(),+ });+ const req = request("/api/public/prompts/[promptName]");++ await call(req);+ await call(req);++ expect(mockShadowAuthDiff).toHaveBeenCalledTimes(2);+ for (const args of mockShadowAuthDiff.mock.calls) {+ expect(args[3]).toEqual({+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET /api/public/prompts/[promptName]",+ });+ }+ });++ it("carries the request resource into every asynchronous per-item check", async () => {+ mockLegacyProjectAuth.mockResolvedValue({ scope });+ mockEnforceAuth.mockResolvedValue({+ success: true,+ scope,+ ctx: authContext(),+ });+ const req = request("/api/public/mcp");+ const result = await call(req);+ expect(result.success).toBe(true);+ if (!result.success) throw new Error("Expected successful legacy auth");+ setRequestMeta(req, {});+ mockShadowAuthDiff.mockClear();++ await Promise.all(+ [1, 2, 3].map(async () => {+ await Promise.resolve();+ shadowAuthorize({+ ctx: result.ctx,+ action: "prompts:CUD",+ resource: { projectId: "prj_1" },+ legacyDecision: { success: true, scope },+ });+ }),+ );++ expect(mockShadowAuthDiff).toHaveBeenCalledTimes(3);+ for (const args of mockShadowAuthDiff.mock.calls) {+ expect(args[0]).toMatchObject({ success: false });+ expect(args[3]).toEqual({+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET /api/public/mcp",+ });+ }+ });++ it("isolates request resources when authentication reuses the same context", async () => {+ mockLegacyProjectAuth.mockResolvedValue({ scope });+ mockEnforceAuth.mockResolvedValue({+ success: true,+ scope,+ ctx: authContext(),+ });+ const first = await call(request("/api/public/mcp"));+ const second = await call(request("/api/public/ingestion"));+ if (!first.success || !second.success) {+ throw new Error("Expected successful legacy auth");+ }+ mockShadowAuthDiff.mockClear();++ for (const result of [first, second]) {+ shadowAuthorize({+ ctx: result.ctx,+ action: "prompts:CUD",+ resource: { projectId: "prj_1" },+ legacyDecision: { success: true, scope },+ });+ }++ expect(+ mockShadowAuthDiff.mock.calls.map((args) => args[3].resource),+ ).toEqual(["GET /api/public/mcp", "GET /api/public/ingestion"]);+ });++ it("never falls back to a concrete URL when route metadata is unavailable", async () => {+ mockLegacyProjectAuth.mockResolvedValue({ scope });+ mockEnforceAuth.mockResolvedValue({+ success: false,+ error: new ForbiddenError("new denial"),+ });++ await call(request());++ expect(mockShadowAuthDiff.mock.calls[0][3]).toEqual({+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET unknown",+ });+ });+});+describe("the migration-mode gate reads a declared env var", () => {const source = (relativeToSrc: string) =>readFileSync(diff --git a/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.tsindex 77c625e2a9..6db1482baf 100644--- a/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts+++ b/web/src/__tests__/server/unit/public-api/shadowAuthDiff.servertest.ts@@ -1,4 +1,6 @@-import { describe, expect, it } from "vitest";+import { afterEach, describe, expect, it, vi } from "vitest";++import { logger } from "@langfuse/shared/src/server";import {__test,@@ -29,6 +31,100 @@ const legacyDeny = {} as LegacyDecision;describe("shadowAuthDiff — the ship-gate signal", () => {+ afterEach(() => vi.restoreAllMocks());++ it("adds the route resource to metrics without duplicating it on the span", () => {+ vi.spyOn(logger, "info").mockImplementation(() => logger);+ const { calls, telemetry } = capture();+ const setAttribute = vi.fn();+ telemetry.span = () => ({ setAttribute });++ shadowAuthDiff(+ deny,+ legacyAllow,+ "prompts:read",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET /api/public/prompts/[promptName]",+ },+ telemetry,+ );++ expect(Object.fromEntries(setAttribute.mock.calls)).toEqual({+ "langfuse.authz.parity.action": "prompts:read",+ "langfuse.authz.parity.access_level": "project",+ "langfuse.authz.parity.result": "new_denies",+ "langfuse.authz.parity.legacy_code": 200,+ "langfuse.authz.parity.new_code": 403,+ });+ expect(calls[0].tags).toEqual({+ action: "prompts:read",+ resource: "GET /api/public/prompts/[promptName]",+ access_level: "project",+ result: "new_denies",+ legacy_code: 200,+ new_code: 403,+ });+ });++ it.each([+ [deny, legacyAllow, "new_denies", 200, 403],+ [allow, legacyDeny, "new_allows", 403, 200],+ ] as const)(+ "logs every mismatch with codes %s / %s without a recording span",+ (newDecision, legacyDecision, result, legacyCode, newCode) => {+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);+ const { telemetry } = capture();+ const metadata = {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "GET /api/public/prompts/[promptName]",+ secretKey: "sk-lf-never-log",+ };+ for (let i = 0; i < 3; i++) {+ shadowAuthDiff(+ newDecision,+ legacyDecision,+ "prompts:read",+ metadata,+ telemetry,+ );+ }+ expect(log).toHaveBeenCalledTimes(3);+ expect(log).toHaveBeenCalledWith("Public API authorization mismatch", {+ event: "authz_parity",+ action: "prompts:read",+ access_level: result === "new_denies" ? "project" : "unknown",+ result,+ legacy_code: legacyCode,+ new_code: newCode,+ api_key_id: "key_1",+ public_key: "pk-lf-1",+ resource: "GET /api/public/prompts/[promptName]",+ });+ },+ );++ it("does not log matches or net-new gates", () => {+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);+ const { telemetry } = capture();+ for (const [newDecision, legacyDecision] of [+ [allow, legacyAllow],+ [deny, legacyDeny],+ [deny, { absent: true }],+ ] as const) {+ shadowAuthDiff(+ newDecision,+ legacyDecision,+ "traces:read",+ { resource: "unknown" },+ telemetry,+ );+ }+ expect(log).not.toHaveBeenCalled();+ });+it.each([["both allow", allow, legacyAllow, "match"],["both deny", deny, legacyDeny, "match"],@@ -38,7 +134,13 @@ describe("shadowAuthDiff — the ship-gate signal", () => {] as const)("%s", (_name, newVerdict, legacyVerdict, result) => {const { calls, telemetry } = capture();expect(- shadowAuthDiff(newVerdict, legacyVerdict, "traces:read", telemetry),+ shadowAuthDiff(+ newVerdict,+ legacyVerdict,+ "traces:read",+ { resource: "unknown" },+ telemetry,+ ),).toBe(result);expect(calls[0].tags.result).toBe(result);});diff --git a/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts b/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.tsindex 78ef6aa29e..4c8274c371 100644--- a/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts+++ b/web/src/__tests__/server/unit/public-api/shadowAuthorize.servertest.ts@@ -136,6 +136,15 @@ describe("shadowAuthorize", () => {});});+ it.each(["legacy", "enforce"])(+ "does not compare per-item decisions in %s even with an auth context",+ (mode) => {+ env.API_AUTH_MIGRATION = mode;+ shadowAuthorize(params("prompts:CUD", authContext([allowPrompts])));+ expect(shadowAuthDiff).not.toHaveBeenCalled();+ },+ );+describe("shadow mode", () => {beforeEach(() => {env.API_AUTH_MIGRATION = "shadow";@@ -153,6 +162,11 @@ describe("shadowAuthorize", () => {{ success: false, error: expect.any(ForbiddenError) },{ success: true, scope: { accessLevel: "project" } },"prompts:CUD",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);});@@ -175,6 +189,11 @@ describe("shadowAuthorize", () => {: { success: false, error: expect.any(ForbiddenError) },legacyDecision,action,+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);},);@@ -185,6 +204,11 @@ describe("shadowAuthorize", () => {{ success: true },{ success: true, scope: { accessLevel: "project" } },"prompts:read",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);});});@@ -243,6 +267,11 @@ describe("protected-label legacy comparisons", () => {{ success: true },{ success: false, error: expect.any(ForbiddenError) },"promptProtectedLabels:CUD",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);} else {expect(shadowAuthDiff).not.toHaveBeenCalled();@@ -269,6 +298,11 @@ describe("protected-label legacy comparisons", () => {{ success: true },{ success: true, scope: { accessLevel: "project" } },"promptProtectedLabels:CUD",+ {+ apiKeyId: "key_1",+ publicKey: "pk-lf-1",+ resource: "unknown",+ },);});diff --git a/web/src/features/public-api/server/shadowAuth.ts b/web/src/features/public-api/server/shadowAuth.tsindex 241c78e132..f7414837d1 100644--- a/web/src/features/public-api/server/shadowAuth.ts+++ b/web/src/features/public-api/server/shadowAuth.ts@@ -1,4 +1,5 @@import { type NextApiRequest } from "next";+import { getRequestMeta } from "next/dist/server/request-meta";import { ApiError, isBaseError, type BaseError } from "@langfuse/shared";import {@@ -45,6 +46,9 @@ import {} from "@/src/features/auth/policy/types";import { isPrismaException } from "@/src/utils/exceptions";+/** shadowResources retains request route templates for asynchronous per-item checks. */+const shadowResources = new WeakMap<AuthorizationContext, string>();+/** shadowAuth authorizes a public-API request under the active migration mode, returning the scope or the error to render. */export async function shadowAuth(params: ShadowAuthParams,@@ -105,14 +109,44 @@ function genericInfrastructureError(httpCode: number): ErrorResult<BaseError> {async function legacyWithShadow(params: ShadowAuthParams,): Promise<ShadowAuthResult> {+ const resource = requestResource(params.req);const legacyAuth = await runLegacyAuth(params);const newAuth = await runNewAuth(params);- shadowAuthDiff(newAuth, legacyAuth, params.action);+ shadowAuthDiff(newAuth, legacyAuth, params.action, {+ ...getScopeIdentity(newAuth, legacyAuth),+ resource,+ });const result = legacyResult(legacyAuth);- if (result.success && newAuth.success) return { ...result, ctx: newAuth.ctx };+ if (result.success && newAuth.success) {+ return { ...result, ctx: withShadowResource(newAuth.ctx, resource) };+ }return result;}+function getScopeIdentity(+ newAuth: EnforceAuthResult,+ legacyAuth: LegacyDecision,+) {+ const successfulAuth = newAuth.success ? newAuth : legacyAuth;+ const scope = successfulAuth.success ? successfulAuth.scope : undefined;+ return { apiKeyId: scope?.apiKeyId, publicKey: scope?.publicKey };+}++function withShadowResource(+ ctx: AuthorizationContext,+ resource: string,+): AuthorizationContext {+ const shadowCtx = { ...ctx };+ shadowResources.set(shadowCtx, resource);+ return shadowCtx;+}++/** requestResource reads Next's matched route independently of recording spans. */+function requestResource(req: NextApiRequest): string {+ const route = getRequestMeta(req, "match")?.definition.pathname;+ return `${req.method ?? "UNKNOWN"} ${route ?? "unknown"}`;+}+/** legacyOnly authorizes solely with the legacy verify. */async function legacyOnly(params: ShadowAuthParams): Promise<ShadowAuthResult> {return legacyResult(await runLegacyAuth(params));@@ -221,12 +255,36 @@ function diffOrGate(decision: Decision,): Decision {if (env.API_AUTH_MIGRATION === "shadow") {- shadowAuthDiff(decision, params.legacyDecision, params.action);+ shadowAuthDiff(+ decision,+ params.legacyDecision,+ params.action,+ getAuthMetadata(params.ctx),+ );return params.legacyDecision;}return decision;}+function getAuthMetadata(ctx: AuthorizationContext | undefined) {+ return { ...getPrincipalIdentity(ctx), resource: getResource(ctx) };+}++function getPrincipalIdentity(ctx: AuthorizationContext | undefined) {+ const principal = ctx?.principal;+ if (principal?.kind === "apiKey") {+ return { apiKeyId: principal.apiKeyId, publicKey: principal.publicKey };+ }+ if (principal?.kind === "admin") {+ return { apiKeyId: "ADMIN_API_KEY", publicKey: "ADMIN_API_KEY" };+ }+ return { apiKeyId: undefined, publicKey: undefined };+}++function getResource(ctx: AuthorizationContext | undefined): string {+ return (ctx && shadowResources.get(ctx)) ?? "unknown";+}+/** toResourceId tags a per-item resource for the PDP. */function toResourceId(resource: Resource): ResourceId {return "projectId" in resourcediff --git a/web/src/features/public-api/server/shadowAuthDiff.ts b/web/src/features/public-api/server/shadowAuthDiff.tsindex 7ecaaf5d9c..1a695b1ce1 100644--- a/web/src/features/public-api/server/shadowAuthDiff.ts+++ b/web/src/features/public-api/server/shadowAuthDiff.ts@@ -1,5 +1,6 @@import {getCurrentSpan,+ logger,recordIncrement,type ApiAccessLevel,} from "@langfuse/shared/src/server";@@ -13,27 +14,57 @@ const defaultTelemetry: Telemetry = {span: () => getCurrentSpan(),};-/** shadowAuthDiff classifies the new pipeline's decision against legacy's, emits `langfuse.authz.parity`, and returns the result. */+/** shadowAuthDiff records parity and logs every mismatch independently of trace sampling. */export function shadowAuthDiff(newDecision: NewDecision,legacyDecision: LegacyDecision,action: string,+ metadata: AuthDiffMetadata,telemetry: Telemetry = defaultTelemetry,): ParityResult {const n = newVerdict(newDecision);const l = legacyVerdict(legacyDecision);const result = classify(l.verdict, n.verdict);- telemetry.increment(parityStat, {+ const tags = {action,access_level: l.accessLevel,result,legacy_code: l.code,new_code: n.code,- });- telemetry.span()?.setAttribute(`${parityStat}.result`, result);+ };+ logMismatch(result, tags, metadata);+ telemetry.increment(parityStat, { ...tags, resource: metadata.resource });+ annotateSpan(telemetry.span(), tags);return result;}+/** logMismatch emits only safe identity and route fields for disagreements. */+function logMismatch(+ result: ParityResult,+ tags: Record<string, string | number>,+ metadata: AuthDiffMetadata,+): void {+ if (result !== "new_allows" && result !== "new_denies") return;+ logger.info("Public API authorization mismatch", {+ event: "authz_parity",+ ...tags,+ api_key_id: metadata.apiKeyId,+ public_key: metadata.publicKey,+ resource: metadata.resource,+ });+}++/** annotateSpan adds decision tags under the parity namespace. */+function annotateSpan(+ span: ReturnType<Telemetry["span"]>,+ tags: Record<string, string | number>,+): void {+ if (!span) return;+ for (const [key, value] of Object.entries(tags)) {+ span.setAttribute(`${parityStat}.${key}`, value);+ }+}+/** newVerdict reads the new decision: success allows (200), else denies with its http code. */function newVerdict(decision: NewDecision): { verdict: Verdict; code: number } {return decision.success@@ -68,6 +99,13 @@ function classify(legacyVerdict: Verdict, newVerdict: Verdict): ParityResult {/** Verdict is one path's decision at one enforcement point; legacy is `absent` only where it runs no gate. */type Verdict = "allow" | "deny" | "absent";+/** AuthDiffMetadata carries safe key identity and the resolved HTTP route resource. */+type AuthDiffMetadata = {+ apiKeyId?: string;+ publicKey?: string;+ resource: string;+};+/** ParityResult is the signal: agreement, which path is stricter, or net-new enforcement legacy never gated. */export type ParityResult = "match" | "new_denies" | "new_allows" | "net_new";