🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(automations): mask and encrypt legacy webhook headers (#19207)
Tobias Wochingera08efe2481a2
diff --git a/packages/shared/prisma/migrations/20261022120000_add_encrypt_legacy_webhook_headers_background_migration/migration.sql b/packages/shared/prisma/migrations/20261022120000_add_encrypt_legacy_webhook_headers_background_migration/migration.sqlnew file mode 100644index 0000000000..b83c5f39b9--- /dev/null+++ b/packages/shared/prisma/migrations/20261022120000_add_encrypt_legacy_webhook_headers_background_migration/migration.sql@@ -0,0 +1,2 @@+INSERT INTO background_migrations (id, name, script, args)+VALUES ('25982425-d431-44db-bbbd-8325b8378a5b', '20261022_encrypt_legacy_webhook_headers', 'encryptLegacyWebhookHeaders', '{}');diff --git a/packages/shared/src/server/repositories/automation-repository.ts b/packages/shared/src/server/repositories/automation-repository.tsindex 1b4522abb6..1efe76c0a0 100644--- a/packages/shared/src/server/repositories/automation-repository.ts+++ b/packages/shared/src/server/repositories/automation-repository.ts@@ -1,4 +1,4 @@-/* eslint-disable no-nested-ternary, @typescript-eslint/no-use-before-define */+/* eslint-disable @typescript-eslint/no-use-before-define */import {Action,ActionExecutionStatus,@@ -29,7 +29,11 @@ import {} from "../../domain/automations";import { InternalServerError } from "../../errors";import { FilterState } from "../../types";-import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";+import {+ createDisplayHeaders,+ decryptSecretHeaders,+ getLegacyOnlyHeaders,+} from "../utils/headerUtils";export const getActionByIdWithSecrets = async ({projectId,@@ -52,20 +56,12 @@ export const getActionByIdWithSecrets = async ({if (isWebhookActionConfig(actionConfig.config)) {const config = actionConfig.config; // Type guard ensures this is WebhookActionConfigWithSecrets- // Decrypt secret headers for webhook execution using new structure- const decryptedHeaders = config.requestHeaders- ? decryptSecretHeaders(- mergeHeaders(config.headers, config.requestHeaders),- )- : config.headers- ? Object.entries(config.headers).reduce(- (acc, [key, value]) => {- acc[key] = { secret: false, value };- return acc;- },- {} as Record<string, { secret: boolean; value: string }>,- )- : {};+ // Legacy headers are plaintext but flagged secret, so they are merged in+ // after decryption.+ const decryptedHeaders = {+ ...getLegacyOnlyHeaders(config),+ ...decryptSecretHeaders(config.requestHeaders ?? {}),+ };return {...actionConfig,@@ -161,19 +157,17 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {};};+/**+ * Display headers are readable by every project role. Legacy plaintext+ * `headers` that `requestHeaders` does not override are always masked, even if+ * a stored `displayHeaders` entry lists them unmasked.+ */const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {- let displayHeaders = config.displayHeaders;- if (!displayHeaders && config.headers) {- // Convert legacy headers to displayHeaders format- displayHeaders = Object.entries(config.headers).reduce(- (acc, [key, value]) => {- acc[key] = { secret: false, value };- return acc;- },- {} as Record<string, { secret: boolean; value: string }>,- );+ const legacyHeaders = createDisplayHeaders(getLegacyOnlyHeaders(config));+ if (!config.displayHeaders && Object.keys(legacyHeaders).length === 0) {+ return config.displayHeaders;}- return displayHeaders;+ return { ...config.displayHeaders, ...legacyHeaders };};/**diff --git a/packages/shared/src/server/utils/headerUtils.ts b/packages/shared/src/server/utils/headerUtils.tsindex 8bf24ffa9e..74df0bf733 100644--- a/packages/shared/src/server/utils/headerUtils.ts+++ b/packages/shared/src/server/utils/headerUtils.ts@@ -51,15 +51,21 @@ export function buildWebhookRequestHeaders({return { headers, sensitiveHeaderNames };}+/**+ * Merges the deprecated plaintext `headers` field into `requestHeaders`.+ * Legacy headers predate the secret flag and commonly carry credentials, so+ * they are treated as secret: masked on display and encrypted on the next+ * write. Their values are plaintext, so `requestHeaders` must already be+ * decrypted (or be passed to `encryptSecretHeaders` afterwards).+ */export function mergeHeaders(legacyHeaders: Record<string, string> = {},requestHeaders: Record<string, { secret: boolean; value: string }> = {},): Record<string, { secret: boolean; value: string }> {const merged: Record<string, { secret: boolean; value: string }> = {};- // Add legacy headers as non-secretfor (const [key, value] of Object.entries(legacyHeaders)) {- merged[key] = { secret: false, value };+ merged[key] = { secret: true, value };}// requestHeaders takes precedence@@ -70,6 +76,26 @@ export function mergeHeaders(return merged;}+/**+ * Legacy plaintext headers that no `requestHeaders` entry overrides, flagged+ * secret. Overridden keys are dropped before any decryption, so a+ * `requestHeaders` value that fails to decrypt never falls back to its stale+ * legacy value.+ */+export function getLegacyOnlyHeaders(config: {+ headers?: Record<string, string>;+ requestHeaders?: RequestHeaders;+}): RequestHeaders {+ const requestHeaders = config.requestHeaders ?? {};+ return mergeHeaders(+ Object.fromEntries(+ Object.entries(config.headers ?? {}).filter(+ ([key]) => !(key in requestHeaders),+ ),+ ),+ );+}+export function createDisplayHeaders(mergedHeaders: Record<string, { secret: boolean; value: string }>,): Record<string, { secret: boolean; value: string }> {diff --git a/web/src/__tests__/server/automations-trpc.servertest.ts b/web/src/__tests__/server/automations-trpc.servertest.tsindex 72a6796860..ba70e612fc 100644--- a/web/src/__tests__/server/automations-trpc.servertest.ts+++ b/web/src/__tests__/server/automations-trpc.servertest.ts@@ -272,10 +272,16 @@ describe("automations trpc", () => {config: {type: "WEBHOOK",url: "https://example.com/webhook",- headers: {- "content-type": "application/json",- "x-api-key": encrypt("secret-api-key-123"),- authorization: encrypt("Bearer secret-token-456"),+ requestHeaders: {+ "content-type": { secret: false, value: "application/json" },+ "x-api-key": {+ secret: true,+ value: encrypt("secret-api-key-123"),+ },+ authorization: {+ secret: true,+ value: encrypt("Bearer secret-token-456"),+ },},displayHeaders: {"content-type": { secret: false, value: "application/json" },@@ -661,7 +667,9 @@ describe("automations trpc", () => {config: {type: "WEBHOOK",url: "https://example.com/webhook",- headers: { "Content-Type": "application/json" },+ requestHeaders: {+ "Content-Type": { secret: false, value: "application/json" },+ },displayHeaders: {"Content-Type": { secret: false, value: "application/json" },},@@ -738,10 +746,10 @@ describe("automations trpc", () => {config: {type: "WEBHOOK",url: "https://example.com/webhook",- headers: {- "content-type": "application/json",- "x-custom": "public-value",- "x-secret": encrypt("secret-value-789"),+ requestHeaders: {+ "content-type": { secret: false, value: "application/json" },+ "x-custom": { secret: false, value: "public-value" },+ "x-secret": { secret: true, value: encrypt("secret-value-789") },},displayHeaders: {"content-type": { secret: false, value: "application/json" },@@ -782,7 +790,7 @@ describe("automations trpc", () => {expect(config).not.toHaveProperty("headers");});- it("should return all legacy header values when reading automation", async () => {+ it("should mask legacy header values when reading automation", async () => {const { project, caller } = await prepare();// Create test trigger and action with legacy headers@@ -811,6 +819,14 @@ describe("automations trpc", () => {"content-type": "application/json","x-api-key": "legacy-api-key-value",},+ requestHeaders: {+ "x-env": { secret: false, value: "production" },+ },+ // A stored displayHeaders entry may list a legacy value unmasked.+ displayHeaders: {+ "x-api-key": { secret: false, value: "legacy-api-key-value" },+ "x-env": { secret: false, value: "production" },+ },apiVersion: { prompt: "v1" },secretKey: encrypt(secretKey),displaySecretKey,@@ -834,16 +850,25 @@ describe("automations trpc", () => {const config = response.action.config as SafeWebhookActionConfig;- // Should have display values with all legacy header values returned- // Legacy headers are converted to the new format with secret: false+ // Legacy headers may hold credentials and are readable by VIEWERs, so+ // they are treated as secret and masked.expect(config.displayHeaders).toEqual({- "content-type": { secret: false, value: "application/json" },- "x-api-key": { secret: false, value: "legacy-api-key-value" },+ "content-type": { secret: true, value: "appl...json" },+ "x-api-key": { secret: true, value: "lega...alue" },+ "x-env": { secret: false, value: "production" },});// Should NOT have raw headers object in responseexpect(config).not.toHaveProperty("headers");expect(config).not.toHaveProperty("requestHeaders");++ const list = await caller.automations.getAutomations({+ projectId: project.id,+ });+ const listed = list.find((a) => a.id === automation.id);+ expect(+ (listed?.action.config as SafeWebhookActionConfig).displayHeaders,+ ).toEqual(config.displayHeaders);});it("should throw error when automation not found", async () => {@@ -1990,6 +2015,87 @@ describe("automations trpc", () => {"x-api-key": { secret: true, value: "new-...-key" },});});++ it.each([+ {+ name: "kept with an empty value",+ requestHeaders: { "x-api-key": { secret: true, value: "" } },+ },+ { name: "omitted (URL-only update)", requestHeaders: undefined },+ ])(+ "should encrypt a legacy header $name on update",+ async ({ requestHeaders }) => {+ const { project, caller } = await prepare();++ const trigger = await prisma.trigger.create({+ data: {+ id: v4(),+ projectId: project.id,+ eventSource: "prompt",+ eventActions: ["created"],+ filter: [],+ status: JobConfigState.ACTIVE,+ },+ });++ const { secretKey, displaySecretKey } = generateWebhookSecret();+ const action = await prisma.action.create({+ data: {+ id: v4(),+ projectId: project.id,+ type: "WEBHOOK",+ config: {+ type: "WEBHOOK",+ url: "https://example.com/webhook",+ headers: { "x-api-key": "legacy-api-key" },+ apiVersion: { prompt: "v1" },+ secretKey: encrypt(secretKey),+ displaySecretKey,+ },+ },+ });++ const automation = await prisma.automation.create({+ data: {+ projectId: project.id,+ triggerId: trigger.id,+ actionId: action.id,+ name: "Legacy Header Encryption Test",+ },+ });++ await caller.automations.updateAutomation({+ projectId: project.id,+ automationId: automation.id,+ name: "Legacy Header Encryption Test",+ eventSource: "prompt",+ eventAction: ["created"],+ filter: [],+ status: JobConfigState.ACTIVE,+ actionType: "WEBHOOK",+ actionConfig: {+ type: "WEBHOOK",+ url: "https://example.com/updated-webhook",+ requestHeaders,+ apiVersion: { prompt: "v1" },+ },+ });++ const updatedAction = await prisma.action.findUnique({+ where: { id: action.id },+ });+ const config = updatedAction?.config as WebhookActionConfigWithSecrets;++ expect(config.headers).toEqual({});+ expect(config.requestHeaders!["x-api-key"].secret).toBe(true);+ expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(+ "legacy-api-key",+ );+ expect(config.displayHeaders).toEqual({+ "x-api-key": { secret: true, value: "lega...-key" },+ });+ },+ );});describe("automations.deleteAutomation", () => {diff --git a/worker/src/__tests__/encryptLegacyWebhookHeaders.test.ts b/worker/src/__tests__/encryptLegacyWebhookHeaders.test.tsnew file mode 100644index 0000000000..24b0a3d733--- /dev/null+++ b/worker/src/__tests__/encryptLegacyWebhookHeaders.test.ts@@ -0,0 +1,124 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { v4 } from "uuid";+import { prisma } from "@langfuse/shared/src/db";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import { type WebhookActionConfigWithSecrets } from "@langfuse/shared";+import { decrypt, encrypt } from "@langfuse/shared/encryption";+import EncryptLegacyWebhookHeaders from "../backgroundMigrations/encryptLegacyWebhookHeaders";++const createLegacyWebhookAction = async (+ projectId: string,+ headerConfig: Partial<WebhookActionConfigWithSecrets> = {+ headers: { "x-api-key": "legacy-api-key" },+ },+) =>+ prisma.action.create({+ data: {+ id: v4(),+ projectId,+ type: "WEBHOOK",+ config: {+ type: "WEBHOOK",+ url: "https://example.com/webhook",+ apiVersion: { prompt: "v1" },+ secretKey: encrypt("whsec"),+ displaySecretKey: "lf-whsec_...",+ ...headerConfig,+ },+ },+ });++describe("EncryptLegacyWebhookHeaders", () => {+ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("encrypts legacy headers without touching headers set in requestHeaders", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const encryptedAuth = encrypt("Bearer new-token");+ const action = await createLegacyWebhookAction(projectId, {+ headers: {+ "X-Legacy": "legacy-value",+ Authorization: "Bearer stale-token",+ },+ requestHeaders: {+ Authorization: { secret: true, value: encryptedAuth },+ },+ displayHeaders: {+ "X-Legacy": { secret: false, value: "legacy-value" },+ Authorization: { secret: true, value: "Bear...oken" },+ },+ });++ await new EncryptLegacyWebhookHeaders().run({});++ const config = (+ await prisma.action.findUniqueOrThrow({ where: { id: action.id } })+ ).config as WebhookActionConfigWithSecrets;+ expect(config.headers).toEqual({});+ expect(config.requestHeaders!.Authorization).toEqual({+ secret: true,+ value: encryptedAuth,+ });+ expect(config.requestHeaders!["X-Legacy"].secret).toBe(true);+ expect(decrypt(config.requestHeaders!["X-Legacy"].value)).toBe(+ "legacy-value",+ );+ expect(config.displayHeaders).toEqual({+ "X-Legacy": { secret: true, value: "lega...alue" },+ Authorization: { secret: true, value: "Bear...oken" },+ });++ // Re-running is a no-op.+ await new EncryptLegacyWebhookHeaders().run({});+ expect(+ (await prisma.action.findUniqueOrThrow({ where: { id: action.id } }))+ .config,+ ).toEqual(config);+ });++ it("retries a row whose config changed concurrently", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const action = await createLegacyWebhookAction(projectId);++ const updateMany = prisma.action.updateMany.bind(prisma.action);+ vi.spyOn(prisma.action, "updateMany").mockImplementationOnce(+ async (args) => {+ // A concurrent writer, e.g. failure tracking, lands first.+ await prisma.$executeRaw`+ UPDATE actions+ SET config = jsonb_set(config, '{lastFailingExecutionId}', '"exec-1"')+ WHERE id = ${action.id}`;+ return updateMany(args);+ },+ );++ await new EncryptLegacyWebhookHeaders().run({});++ const config = (+ await prisma.action.findUniqueOrThrow({ where: { id: action.id } })+ ).config as WebhookActionConfigWithSecrets;+ expect(config.headers).toEqual({});+ expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(+ "legacy-api-key",+ );+ expect(config.lastFailingExecutionId).toBe("exec-1");+ });++ it("fails the migration when a row cannot be encrypted", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const action = await createLegacyWebhookAction(projectId);++ vi.spyOn(prisma.action, "updateMany").mockRejectedValue(+ new Error("connection lost"),+ );++ await expect(new EncryptLegacyWebhookHeaders().run({})).rejects.toThrow(+ action.id,+ );+ const config = (+ await prisma.action.findUniqueOrThrow({ where: { id: action.id } })+ ).config as WebhookActionConfigWithSecrets;+ expect(config.headers).toEqual({ "x-api-key": "legacy-api-key" });+ });+});diff --git a/worker/src/__tests__/webhooks.test.ts b/worker/src/__tests__/webhooks.test.tsindex 5356a78319..b5b1841fb2 100644--- a/worker/src/__tests__/webhooks.test.ts+++ b/worker/src/__tests__/webhooks.test.ts@@ -2237,3 +2237,39 @@ describe("Webhook Integration Tests", () => {});});});++describe("getActionByIdWithSecrets legacy headers", () => {+ it("sends legacy headers as secret and never falls back to a legacy value its requestHeaders entry overrides", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const action = await prisma.action.create({+ data: {+ id: v4(),+ projectId,+ type: "WEBHOOK",+ config: {+ type: "WEBHOOK",+ url: "https://example.com/webhook",+ headers: {+ "x-api-key": "legacy-api-key",+ Authorization: "Bearer stale-token",+ },+ requestHeaders: {+ Authorization: { secret: true, value: "not-a-valid-ciphertext" },+ },+ apiVersion: { prompt: "v1" },+ secretKey: encrypt("whsec"),+ displaySecretKey: "lf-whsec_...",+ },+ },+ });++ const result = await getActionByIdWithSecrets({+ projectId,+ actionId: action.id,+ });++ expect(+ (result?.config as WebhookActionConfigWithSecrets).requestHeaders,+ ).toEqual({ "x-api-key": { secret: true, value: "legacy-api-key" } });+ });+});diff --git a/worker/src/backgroundMigrations/encryptLegacyWebhookHeaders.ts b/worker/src/backgroundMigrations/encryptLegacyWebhookHeaders.tsnew file mode 100644index 0000000000..d9f1239ded--- /dev/null+++ b/worker/src/backgroundMigrations/encryptLegacyWebhookHeaders.ts@@ -0,0 +1,183 @@+import { IBackgroundMigration } from "./IBackgroundMigration";+import {+ createDisplayHeaders,+ encryptSecretHeaders,+ getLegacyOnlyHeaders,+ logger,+} from "@langfuse/shared/src/server";+import { Prisma, prisma } from "@langfuse/shared/src/db";+import {+ isWebhookActionConfig,+ type WebhookActionConfigWithSecrets,+} from "@langfuse/shared";++const MAX_ATTEMPTS_PER_ACTION = 5;++type Outcome = "encrypted" | "unchanged" | "invalid" | "failed";++function hasLegacyHeaders(config: Prisma.JsonValue): boolean {+ if (typeof config !== "object" || config === null || Array.isArray(config)) {+ return false;+ }+ const headers = config.headers;+ return (+ typeof headers === "object" &&+ headers !== null &&+ Object.keys(headers).length > 0+ );+}++function withEncryptedLegacyHeaders(+ config: WebhookActionConfigWithSecrets,+): WebhookActionConfigWithSecrets {+ const legacyHeaders = getLegacyOnlyHeaders(config);+ return {+ ...config,+ headers: {},+ requestHeaders: {+ ...encryptSecretHeaders(legacyHeaders),+ ...config.requestHeaders,+ },+ displayHeaders: {+ ...config.displayHeaders,+ ...createDisplayHeaders(legacyHeaders),+ },+ };+}++async function encryptLegacyHeadersOfAction(action: {+ id: string;+ config: Prisma.JsonValue;+}): Promise<Outcome> {+ let config = action.config;+ try {+ for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_ACTION; attempt++) {+ if (!hasLegacyHeaders(config)) return "unchanged";+ if (!isWebhookActionConfig(config)) return "invalid";++ const { count } = await prisma.action.updateMany({+ where: { id: action.id, config: { equals: config } },+ data: { config: withEncryptedLegacyHeaders(config) },+ });+ if (count === 1) return "encrypted";++ const current = await prisma.action.findUnique({+ where: { id: action.id },+ select: { config: true },+ });+ if (!current) return "unchanged";+ config = current.config;+ }+ } catch (error) {+ logger.error(+ `[Background Migration] Failed to encrypt legacy webhook headers for action ${action.id}`,+ { error },+ );+ }+ return "failed";+}++/**+ * Legacy headers were only written for a short window before header+ * encryption existed, so this set is small and loaded in one query.+ */+function findActionsWithLegacyHeaders() {+ return prisma.$queryRaw<{ id: string; config: Prisma.JsonValue }[]>`+ SELECT id, config+ FROM actions+ WHERE type = 'WEBHOOK'+ AND jsonb_typeof(config -> 'headers') = 'object'+ AND config -> 'headers' <> '{}'::jsonb+ ORDER BY id`;+}++/**+ * Background migration that encrypts webhook headers stored in the deprecated+ * plaintext `config.headers` field.+ *+ * Legacy headers predate the secret flag and commonly carry credentials. Each+ * one not already overridden by `requestHeaders` is moved into+ * `requestHeaders` as an encrypted secret, its masked value is added to+ * `displayHeaders`, and `headers` is emptied.+ *+ * Rows are written with a compare-and-swap on the whole config, so concurrent+ * writers (form saves, secret rotation, failure tracking) are never+ * overwritten; a lost race re-reads the row and retries. The migration throws+ * if any convertible row is left in plaintext, so it is marked failed rather+ * than finished. It is idempotent and can be safely re-run.+ */+export default class EncryptLegacyWebhookHeaders implements IBackgroundMigration {+ private isAborted = false;++ async validate(+ _args: Record<string, unknown>,+ ): Promise<{ valid: boolean; invalidReason: string | undefined }> {+ return { valid: true, invalidReason: undefined };+ }++ async run(_args: Record<string, unknown>): Promise<void> {+ const startTime = Date.now();+ logger.info(+ "[Background Migration] Starting legacy webhook header encryption",+ );++ let encrypted = 0;+ const invalidIds: string[] = [];+ const failedIds: string[] = [];++ for (const action of await findActionsWithLegacyHeaders()) {+ if (this.isAborted) {+ logger.info(+ `[Background Migration] Legacy webhook header encryption aborted after encrypting ${encrypted} actions`,+ );+ return;+ }+ const outcome = await encryptLegacyHeadersOfAction(action);+ if (outcome === "encrypted") encrypted++;+ if (outcome === "invalid") invalidIds.push(action.id);+ if (outcome === "failed") failedIds.push(action.id);+ }++ if (invalidIds.length > 0) {+ logger.warn(+ `[Background Migration] Skipped ${invalidIds.length} webhook actions with legacy headers whose config does not parse: ${invalidIds.join(", ")}`,+ );+ }+ logger.info(+ `[Background Migration] Legacy webhook header encryption finished in ${Date.now() - startTime}ms: ` ++ `${encrypted} encrypted, ${invalidIds.length} invalid, ${failedIds.length} failed`,+ );+ if (failedIds.length > 0) {+ throw new Error(+ `Failed to encrypt legacy headers of webhook actions: ${failedIds.join(", ")}`,+ );+ }+ }++ async abort(): Promise<void> {+ logger.info(+ "[Background Migration] Aborting EncryptLegacyWebhookHeaders migration",+ );+ this.isAborted = true;+ }+}++async function main() {+ const migration = new EncryptLegacyWebhookHeaders();+ await migration.validate({});+ await migration.run({});+}++if (require.main === module) {+ main()+ .then(() => {+ process.exit(0);+ })+ .catch((error) => {+ logger.error(+ `[Background Migration] Migration execution failed: ${error}`,+ error,+ );+ process.exit(1);+ });+}