🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import dns from "node:dns/promises";
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type * as SharedServer from "@langfuse/shared/src/server";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { type Prisma, prisma } from "@langfuse/shared/src/db";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import {
createOrgProjectAndApiKey,
SLACK_CHANNEL_NOT_FOUND_MESSAGE,
} from "@langfuse/shared/src/server";
import type { Session } from "next-auth";
import { v4 } from "uuid";
import {
ActionExecutionStatus,
JobConfigState,
type SafeWebhookActionConfig,
TriggerEventSource,
type WebhookActionConfigWithSecrets,
isWebhookAction,
} from "@langfuse/shared";
import {
encrypt,
decrypt,
generateWebhookSecret,
} from "@langfuse/shared/encryption";
import { TRPCError } from "@trpc/server";
import { type MockInstance } from "vitest";
// Fake Slack workspace: the bot is in private channels "G_PRIVATE" and archived
// "G_ARCHIVED". Slack API calls are faked; the access logic is real.
const fakeSlack = vi.hoisted(() => ({
installerChannelIds: ["G_PRIVATE", "G_ARCHIVED"],
missingScope: false,
}));
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual<typeof SharedServer>(
"@langfuse/shared/src/server",
);
const channels = [
{ id: "C123456", name: "general", is_private: false },
{ id: "G_PRIVATE", name: "secret-plans", is_private: true },
{
id: "G_ARCHIVED",
name: "old-secrets",
is_private: true,
is_archived: true,
},
];
const client = {
users: {
conversations: async (args: {
user?: string;
exclude_archived?: boolean;
}) => {
if (fakeSlack.missingScope) {
throw Object.assign(new Error("missing_scope"), {
data: { error: "missing_scope" },
});
}
return {
ok: true,
channels: channels.filter(
(channel) =>
channel.is_private &&
!(args.exclude_archived && channel.is_archived) &&
(!args.user ||
(args.user === "U_INSTALLER" &&
fakeSlack.installerChannelIds.includes(channel.id))),
),
};
},
},
conversations: {
info: async ({ channel }: { channel: string }) => {
const found = channels.find(({ id }) => id === channel);
if (!found) throw new Error("channel_not_found");
return { ok: true, channel: found };
},
},
};
const slackService = {
getWebClientForProject: async () => client,
getPrivateChannels: actual.SlackService.prototype.getPrivateChannels,
checkPrivateChannelAccess:
actual.SlackService.prototype.checkPrivateChannelAccess,
getChannelInfo: actual.SlackService.prototype.getChannelInfo,
};
// Patched on the real class so calls from inside the shared module see it too.
actual.SlackService.getInstance = () =>
slackService as unknown as SharedServer.SlackService;
return actual;
});
const __orgIds: string[] = [];
async function prepare() {
const { project, org } = await createOrgProjectAndApiKey();
const session: Session = {
expires: "1",
user: {
id: "user-1",
canCreateOrganizations: true,
name: "Demo User",
organizations: [
{
id: org.id,
name: org.name,
role: "OWNER",
plan: "cloud:hobby",
cloudConfig: undefined,
metadata: {},
aiFeaturesEnabled: false,
aiTelemetryEnabled: false,
projects: [
{
id: project.id,
role: "ADMIN",
retentionDays: 30,
deletedAt: null,
hasTraces: false,
name: project.name,
metadata: {},
createdAt: new Date().toISOString(),
},
],
},
],
featureFlags: testFeatureFlags(),
admin: true,
},
environment: {
enableExperimentalFeatures: false,
selfHostedInstancePlan: "cloud:hobby",
},
};
const ctx = createInnerTRPCContext({ session, headers: {} });
const caller = appRouter.createCaller({ ...ctx, prisma });
__orgIds.push(org.id);
return { project, org, session, ctx, caller };
}
describe("automations trpc", () => {
afterAll(async () => {
await prisma.organization.deleteMany({
where: {
id: { in: __orgIds },
},
});
});
describe("automations.getAutomations", () => {
it("should retrieve all automations for a project", async () => {
const { project, caller } = await prepare();
// Create test prompt
await prisma.prompt.create({
data: {
id: v4(),
projectId: project.id,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello world" },
createdBy: "test-user",
},
});
// Create test trigger
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
// Create test action
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
headers: { "Content-Type": "application/json" },
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
// Link trigger to action
await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
const response = await caller.automations.getAutomations({
projectId: project.id,
});
expect(response).toHaveLength(1);
expect(response[0]).toMatchObject({
name: "Test Automation",
trigger: expect.objectContaining({
id: trigger.id,
eventSource: "prompt",
eventActions: ["created"],
status: JobConfigState.ACTIVE,
}),
action: expect.objectContaining({
id: action.id,
type: "WEBHOOK",
config: expect.objectContaining({
type: "WEBHOOK",
url: "https://example.com/webhook",
displaySecretKey,
}),
}),
});
expect(response[0].action.config).not.toHaveProperty("secretKey");
});
it("should not expose secret headers in response", async () => {
const { project, caller } = await prepare();
// Create test prompt
await prisma.prompt.create({
data: {
id: v4(),
projectId: project.id,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello world" },
createdBy: "test-user",
},
});
// Create test trigger
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
// Create test action with secret headers
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-api-key": {
secret: true,
value: encrypt("secret-api-key-123"),
},
authorization: {
secret: true,
value: encrypt("Bearer secret-token-456"),
},
},
displayHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "secr***123" },
authorization: { secret: true, value: "Bear***456" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
// Link trigger to action
await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Secret Headers Automation",
},
});
const response = await caller.automations.getAutomations({
projectId: project.id,
});
expect(response).toHaveLength(1);
const automationConfig = response[0].action
.config as SafeWebhookActionConfig;
// Should have display values, not encrypted values
expect(automationConfig.displayHeaders).toEqual({
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "secr***123" },
authorization: { secret: true, value: "Bear***456" },
});
// Should NOT have the raw headers with encrypted values
expect(automationConfig).not.toHaveProperty("headers");
expect(automationConfig).not.toHaveProperty("decryptedHeaders");
expect(automationConfig).not.toHaveProperty("requestHeaders");
});
it("should return empty array when no automations exist", async () => {
const { project, caller } = await prepare();
const response = await caller.automations.getAutomations({
projectId: project.id,
});
expect(response).toEqual([]);
});
it("should return empty array for user with read access but no automations", async () => {
const { project, session } = await prepare();
// Create a session with limited permissions (VIEWER can read but not create)
const limitedSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: "VIEWER", // VIEWER role can read automations but not create/update/delete
},
],
},
],
},
};
const limitedCtx = createInnerTRPCContext({
session: limitedSession,
headers: {},
});
const limitedCaller = appRouter.createCaller({ ...limitedCtx, prisma });
const response = await limitedCaller.automations.getAutomations({
projectId: project.id,
});
expect(response).toEqual([]);
});
describe("eventSource narrowing", () => {
async function createAutomation(opts: {
projectId: string;
name: string;
eventSource: "prompt" | "monitor";
filter: unknown[];
eventActions?: string[];
}) {
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: opts.projectId,
eventSource: opts.eventSource,
eventActions: opts.eventActions ?? [],
filter: opts.filter as object,
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: opts.projectId,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
await prisma.automation.create({
data: {
projectId: opts.projectId,
triggerId: trigger.id,
actionId: action.id,
name: opts.name,
},
});
}
it("filters by eventSource at the DB layer", async () => {
const { project, caller } = await prepare();
await createAutomation({
projectId: project.id,
name: "prompt-automation",
eventSource: "prompt",
filter: [],
});
await createAutomation({
projectId: project.id,
name: "monitor-automation",
eventSource: "monitor",
filter: [],
});
const monitor = await caller.automations.getAutomations({
projectId: project.id,
eventSource: TriggerEventSource.Monitor,
});
expect(monitor.map((a) => a.name)).toEqual(["monitor-automation"]);
const prompt = await caller.automations.getAutomations({
projectId: project.id,
eventSource: TriggerEventSource.Prompt,
});
expect(prompt.map((a) => a.name)).toEqual(["prompt-automation"]);
});
});
describe("project-notification eventSource", () => {
it("lists project-notification automations for any reader", async () => {
const { project, caller } = await prepare();
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "project-notification",
eventActions: ["blob-export-failed"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
apiVersion: { "project-notification": "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "notification-channel",
},
});
const response = await caller.automations.getAutomations({
projectId: project.id,
eventSource: TriggerEventSource.ProjectNotification,
});
expect(response.map((a) => a.name)).toEqual(["notification-channel"]);
});
it("excludes project-notification automations from the general list and count", async () => {
const { project, caller } = await prepare();
// A prompt automation (general UI) plus a project-notification channel.
const { secretKey, displaySecretKey } = generateWebhookSecret();
const sharedConfig = {
type: "WEBHOOK" as const,
url: "https://example.com/webhook",
apiVersion: { prompt: "v1" as const },
secretKey: encrypt(secretKey),
displaySecretKey,
};
for (const [eventSource, name, apiVersion] of [
["prompt", "prompt-automation", { prompt: "v1" as const }],
[
"project-notification",
"notification-channel",
{ "project-notification": "v1" as const },
],
] as const) {
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource,
eventActions:
eventSource === "prompt" ? ["created"] : ["blob-export-failed"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: { ...sharedConfig, apiVersion },
},
});
await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name,
},
});
}
// No eventSource → general list excludes project-notification.
const generalList = await caller.automations.getAutomations({
projectId: project.id,
});
expect(generalList.map((a) => a.name)).toEqual(["prompt-automation"]);
// Explicit eventSource still returns the channel.
const notificationList = await caller.automations.getAutomations({
projectId: project.id,
eventSource: TriggerEventSource.ProjectNotification,
});
expect(notificationList.map((a) => a.name)).toEqual([
"notification-channel",
]);
// Header count excludes the project-notification channel.
const count = await caller.automations.count({
projectId: project.id,
});
expect(count).toBe(1);
});
it("toggles trigger eventActions via updateTriggerEventActions, only for project-notification sources", async () => {
const { project, caller } = await prepare();
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "project-notification",
eventActions: ["blob-export-failed", "evaluator-blocked"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
apiVersion: { "project-notification": "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "notification-channel",
},
});
await caller.automations.updateTriggerEventActions({
projectId: project.id,
automationId: automation.id,
eventActions: ["blob-export-failed"],
});
const updatedTrigger = await prisma.trigger.findUniqueOrThrow({
where: { id: trigger.id },
});
expect(updatedTrigger.eventActions).toEqual(["blob-export-failed"]);
// Prompt-source automations must be rejected.
const promptTrigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const promptAutomation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: promptTrigger.id,
actionId: action.id,
name: "prompt-automation",
},
});
await expect(
caller.automations.updateTriggerEventActions({
projectId: project.id,
automationId: promptAutomation.id,
eventActions: [],
}),
).rejects.toThrow(
"This operation is only supported on project-notification automations.",
);
});
});
});
describe("automations.getAutomation", () => {
it("should retrieve a specific automation", async () => {
const { project, caller } = await prepare();
// Create test trigger and action
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
displayHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Specific Automation",
},
});
const response = await caller.automations.getAutomation({
projectId: project.id,
automationId: automation.id,
});
expect(response).toMatchObject({
name: "Specific Automation",
trigger: expect.objectContaining({
id: trigger.id,
}),
action: expect.objectContaining({
id: action.id,
}),
});
const actionConfig = response.action.config as SafeWebhookActionConfig;
// check that the action does not have a secret key in the config
expect(actionConfig).not.toHaveProperty("secretKey");
expect(actionConfig).toHaveProperty("displaySecretKey");
expect(actionConfig.url).toBe("https://example.com/webhook");
expect(actionConfig).not.toHaveProperty("headers");
expect(actionConfig.displayHeaders).toEqual({
"Content-Type": { secret: false, value: "application/json" },
});
expect(actionConfig.apiVersion).toEqual({ prompt: "v1" });
expect(actionConfig.type).toBe("WEBHOOK");
expect(actionConfig.displaySecretKey).toBe(displaySecretKey);
expect(actionConfig).not.toHaveProperty("headers");
expect(actionConfig).not.toHaveProperty("decryptedHeaders");
expect(actionConfig).not.toHaveProperty("requestHeaders");
});
it("should not expose secret headers in single automation response", async () => {
const { project, caller } = await prepare();
// Create test trigger and action with secret headers
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-custom": { secret: false, value: "public-value" },
"x-secret": { secret: true, value: encrypt("secret-value-789") },
},
displayHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-custom": { secret: false, value: "public-value" },
"x-secret": { secret: true, value: "secr***789" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Mixed Headers Automation",
},
});
const response = await caller.automations.getAutomation({
projectId: project.id,
automationId: automation.id,
});
const config = response.action.config as SafeWebhookActionConfig;
// Should have display values
expect(config.displayHeaders).toEqual({
"content-type": { secret: false, value: "application/json" },
"x-custom": { secret: false, value: "public-value" },
"x-secret": { secret: true, value: "secr***789" },
});
// Should NOT have raw encrypted headers
expect(config).not.toHaveProperty("headers");
});
it("should mask legacy header values when reading automation", async () => {
const { project, caller } = await prepare();
// Create test trigger and action with legacy headers
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
// Legacy headers format - plain object with string values
headers: {
"content-type": "application/json",
"x-api-key": "legacy-api-key-value",
},
requestHeaders: {
"x-env": { secret: false, value: "production" },
},
// A stored displayHeaders entry may list a legacy value unmasked.
displayHeaders: {
"x-api-key": { secret: false, value: "legacy-api-key-value" },
"x-env": { secret: false, value: "production" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Legacy Headers Read Test",
},
});
const response = await caller.automations.getAutomation({
projectId: project.id,
automationId: automation.id,
});
const config = response.action.config as SafeWebhookActionConfig;
// Legacy headers may hold credentials and are readable by VIEWERs, so
// they are treated as secret and masked.
expect(config.displayHeaders).toEqual({
"content-type": { secret: true, value: "appl...json" },
"x-api-key": { secret: true, value: "lega...alue" },
"x-env": { secret: false, value: "production" },
});
// Should NOT have raw headers object in response
expect(config).not.toHaveProperty("headers");
expect(config).not.toHaveProperty("requestHeaders");
const list = await caller.automations.getAutomations({
projectId: project.id,
});
const listed = list.find((a) => a.id === automation.id);
expect(
(listed?.action.config as SafeWebhookActionConfig).displayHeaders,
).toEqual(config.displayHeaders);
});
it("should throw error when automation not found", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.getAutomation({
projectId: project.id,
automationId: "non-existent-automation",
}),
).rejects.toThrow(
`Automation with id non-existent-automation not found.`,
);
});
});
describe("automations.createAutomation", () => {
it("should create a new webhook automation", async () => {
const { project, caller } = await prepare();
// Create test prompt
await prisma.prompt.create({
data: {
id: v4(),
projectId: project.id,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello world" },
createdBy: "test-user",
},
});
const response = await caller.automations.createAutomation({
projectId: project.id,
name: "New Webhook Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/new-webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
},
});
expect(response.trigger).toMatchObject({
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
status: JobConfigState.ACTIVE,
});
expect(response.action).toMatchObject({
projectId: project.id,
type: "WEBHOOK",
config: expect.objectContaining({
type: "WEBHOOK",
url: "https://example.com/new-webhook",
}),
});
expect(response.webhookSecret).toBeDefined();
expect(typeof response.webhookSecret).toBe("string");
// Verify the automation link was created
const automation = await prisma.automation.findFirst({
where: {
triggerId: response.trigger.id,
actionId: response.action.id,
},
});
expect(automation).toMatchObject({
name: "New Webhook Automation",
});
});
it("should create automation with both plain and secret headers", async () => {
const { project, caller } = await prepare();
// Create test prompt
await prisma.prompt.create({
data: {
id: v4(),
projectId: project.id,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello world" },
createdBy: "test-user",
},
});
const response = await caller.automations.createAutomation({
projectId: project.id,
name: "Mixed Headers Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/mixed-headers",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
"x-api-key": { secret: true, value: "secret-key-123" },
authorization: { secret: true, value: "Bearer secret-token-456" },
},
apiVersion: { prompt: "v1" },
},
});
// Verify the action was created correctly in the database
const createdAction = await prisma.action.findUnique({
where: { id: response.action.id },
});
expect(createdAction?.config).toMatchObject({
type: "WEBHOOK",
url: "https://example.com/mixed-headers",
});
expect(isWebhookAction(createdAction as any)).toBe(true);
// Headers should be encrypted for secret ones, plain for others
const config = createdAction?.config as WebhookActionConfigWithSecrets;
expect(config.requestHeaders!["content-type"].value).toBe(
"application/json",
);
expect(config.requestHeaders!["x-public"].value).toBe("public-value");
expect(config.requestHeaders!["x-api-key"].secret).toBe(true);
expect(config.requestHeaders!["x-api-key"].value).not.toBe(
"secret-key-123",
); // Should be encrypted
expect(config.requestHeaders!["authorization"].secret).toBe(true);
expect(config.requestHeaders!["authorization"].value).not.toBe(
"Bearer secret-token-456",
); // Should be encrypted
// Display values should be present
expect(config.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
});
expect(config.displayHeaders!["x-api-key"].value).toBe("secr...-123");
expect(config.displayHeaders!["authorization"].value).toBe("Bear...-456");
});
it("should create automation with secret headers that do not expose values in response", async () => {
const { project, caller } = await prepare();
const response = await caller.automations.createAutomation({
projectId: project.id,
name: "Secret Headers Test",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
"x-api-key": { secret: true, value: "secret-value-123" },
authorization: { secret: true, value: "Bearer token-456" },
},
apiVersion: { prompt: "v1" },
},
});
const responseConfig = response.action.config as SafeWebhookActionConfig;
// Response should NOT contain the raw secret values
expect(responseConfig.displayHeaders).not.toMatchObject({
"x-api-key": { secret: true, value: "secret-value-123" },
authorization: { secret: true, value: "Bearer token-456" },
});
// Response should contain masked values
expect(responseConfig.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
"x-api-key": { secret: true, value: "secr...-123" },
authorization: { secret: true, value: "Bear...-456" },
});
// Verify secrets fields are not present in response
expect(responseConfig).not.toHaveProperty("headers");
expect(responseConfig).not.toHaveProperty("requestHeaders");
// Check the actual stored data in the database
const createdAction = await prisma.action.findUnique({
where: { id: response.action.id },
});
const config = createdAction?.config as WebhookActionConfigWithSecrets;
// Public headers should remain plain
// Secret headers should be encrypted in storage
expect(config.requestHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
});
expect(config.requestHeaders!["x-api-key"].value).not.toBe(
"secret-value-123",
);
expect(config.requestHeaders!["authorization"].value).not.toBe(
"Bearer token-456",
);
// Display values should be present with masked secrets
expect(config.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "public-value" },
"x-api-key": { secret: true, value: "secr...-123" },
authorization: { secret: true, value: "Bear...-456" },
});
});
it("should throw error when user lacks automations:CUD access", async () => {
const { project, session } = await prepare();
// Create a session with limited permissions
const limitedSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: "VIEWER", // VIEWER role doesn't have automations:CUD scope
},
],
},
],
},
};
const limitedCtx = createInnerTRPCContext({
session: limitedSession,
headers: {},
});
const limitedCaller = appRouter.createCaller({ ...limitedCtx, prisma });
await expect(
limitedCaller.automations.createAutomation({
projectId: project.id,
name: "Unauthorized Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {},
apiVersion: { prompt: "v1" },
},
}),
).rejects.toThrow("User does not have access to this resource or action");
});
it("should validate required fields", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation({
projectId: project.id,
name: "", // Empty name should fail
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {},
apiVersion: { prompt: "v1" },
},
}),
).rejects.toThrow("Name is required");
});
describe("monitor webhook destination validation", () => {
const original = {
hosts: sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_HOST,
ips: sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IPS,
ipSegments: sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IP_SEGMENTS,
allowedHosts: sharedEnv.LANGFUSE_WEBHOOK_ALLOWED_HOSTS,
};
const monitorWebhookInput = (projectId: string, url: string) => ({
projectId,
name: "Monitor webhook",
eventSource: TriggerEventSource.Monitor,
eventAction: [],
filter: null,
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK" as const,
actionConfig: {
type: "WEBHOOK" as const,
url,
requestHeaders: {},
apiVersion: { monitor: "v1" as const },
},
});
async function expectNothingPersisted(projectId: string) {
const [automations, triggers, actions] = await Promise.all([
prisma.automation.count({ where: { projectId } }),
prisma.trigger.count({ where: { projectId } }),
prisma.action.count({ where: { projectId } }),
]);
expect({ automations, triggers, actions }).toEqual({
automations: 0,
triggers: 0,
actions: 0,
});
}
beforeEach(() => {
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_HOST = [];
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IPS = [];
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IP_SEGMENTS = [];
sharedEnv.LANGFUSE_WEBHOOK_ALLOWED_HOSTS = [];
});
afterEach(() => {
vi.restoreAllMocks();
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_HOST = original.hosts;
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IPS = original.ips;
sharedEnv.LANGFUSE_WEBHOOK_WHITELISTED_IP_SEGMENTS =
original.ipSegments;
sharedEnv.LANGFUSE_WEBHOOK_ALLOWED_HOSTS = original.allowedHosts;
});
it.each([
["loopback IPv6", "http://[::1]/hook", "Blocked IP address"],
[
"cloud metadata",
"http://169.254.169.254/latest/meta-data",
"Blocked hostname",
],
["RFC1918", "http://10.0.0.1/hook", "Blocked IP address"],
[
"embedded credentials",
"https://user:[email protected]/hook",
"URL credentials are not allowed",
],
])(
"rejects a %s destination without persisting it",
async (_case, url, message) => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation(
monitorWebhookInput(project.id, url),
),
).rejects.toMatchObject({
code: "BAD_REQUEST",
message: expect.stringContaining(`Invalid webhook URL: ${message}`),
});
await expectNothingPersisted(project.id);
},
);
it("rejects a hostname that resolves to a private address", async () => {
vi.spyOn(dns, "resolve4").mockResolvedValue(["10.0.0.5"]);
vi.spyOn(dns, "resolve6").mockRejectedValue(new Error("ENODATA"));
vi.spyOn(dns, "lookup").mockResolvedValue([
{ address: "10.0.0.5", family: 4 },
] as never);
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation(
monitorWebhookInput(
project.id,
"https://hooks.internal.example.com",
),
),
).rejects.toMatchObject({
code: "BAD_REQUEST",
message: expect.stringContaining(
"Invalid webhook URL: Blocked IP address",
),
});
expect(dns.resolve4).toHaveBeenCalledWith("hooks.internal.example.com");
await expectNothingPersisted(project.id);
});
});
it("should create a new Slack automation", async () => {
const { project, caller } = await prepare();
// Create Slack integration first
await prisma.slackIntegration.create({
data: {
projectId: project.id,
teamId: "T123456",
teamName: "Test Team",
botToken: encrypt("xoxb-test-token"),
botUserId: "U123456",
},
});
const response = await caller.automations.createAutomation({
projectId: project.id,
name: "New Slack Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "SLACK",
actionConfig: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
messageTemplate: JSON.stringify([
{
type: "section",
text: { type: "mrkdwn", text: "Custom template" },
},
]),
},
});
expect(response.trigger).toMatchObject({
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
status: JobConfigState.ACTIVE,
});
expect(response.action).toMatchObject({
projectId: project.id,
type: "SLACK",
config: expect.objectContaining({
type: "SLACK",
channelId: "C123456",
channelName: "general",
}),
});
// Ensure no bot token is exposed in response
expect(JSON.stringify(response)).not.toContain("xoxb-");
});
it("should fail to create Slack automation without integration", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation({
projectId: project.id,
name: "Invalid Slack Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "SLACK",
actionConfig: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
},
}),
).rejects.toThrow(
"Slack integration not found. Please connect your Slack workspace first.",
);
});
});
describe("Slack private channels", () => {
const slackAutomationInput = (projectId: string, channelId: string) => ({
projectId,
name: "Slack Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "SLACK" as const,
actionConfig: {
type: "SLACK" as const,
channelId,
channelName: "secret-plans",
},
});
const prepareWithInstaller = async (installerUserId: string) => {
const prepared = await prepare();
await prisma.slackIntegration.create({
data: {
projectId: prepared.project.id,
teamId: "T123456",
teamName: "Test Team",
botToken: encrypt("xoxb-test-token"),
botUserId: "U123456",
installerUserId,
installerSlackUserId: "U_INSTALLER",
},
});
const otherUserCaller = appRouter.createCaller({
...createInnerTRPCContext({
session: {
...prepared.session,
user: { ...prepared.session.user!, id: "user-2" },
},
headers: {},
}),
prisma,
});
return { ...prepared, otherUserCaller };
};
beforeEach(() => {
fakeSlack.installerChannelIds = ["G_PRIVATE", "G_ARCHIVED"];
fakeSlack.missingScope = false;
});
it("rejects private channels the user can't use exactly like non-existent channels", async () => {
const { project, caller } = await prepareWithInstaller("someone-else");
const errorFor = (channelId: string) =>
caller.automations
.createAutomation(slackAutomationInput(project.id, channelId))
.catch((error: Error) => error.message);
for (const channelId of ["G_PRIVATE", "G_ARCHIVED", "C_MISSING"]) {
expect(await errorFor(channelId)).toBe(SLACK_CHANNEL_NOT_FOUND_MESSAGE);
}
// Typed names are rejected before any lookup.
expect(await errorFor("#secret-plans")).toBe(
await errorFor("#does-not-exist"),
);
});
it("fails closed for private channels on installs without groups:read", async () => {
const { project, caller } = await prepareWithInstaller("user-1");
fakeSlack.missingScope = true;
await expect(
caller.automations.createAutomation(
slackAutomationInput(project.id, "G_PRIVATE"),
),
).rejects.toThrow(SLACK_CHANNEL_NOT_FOUND_MESSAGE);
const created = await caller.automations.createAutomation(
slackAutomationInput(project.id, "C123456"),
);
expect(created.action.config).toMatchObject({
channelName: "general",
isPrivate: false,
});
});
it("masks a saved private channel name for everyone but the installer", async () => {
const { project, caller, otherUserCaller } =
await prepareWithInstaller("user-1");
const created = await caller.automations.createAutomation(
slackAutomationInput(project.id, "G_PRIVATE"),
);
expect(created.action.config).toMatchObject({
channelId: "G_PRIVATE",
channelName: "secret-plans",
isPrivate: true,
});
const [otherList, otherSingle] = await Promise.all([
otherUserCaller.automations.getAutomations({ projectId: project.id }),
otherUserCaller.automations.getAutomation({
projectId: project.id,
automationId: created.automation.id,
}),
]);
expect(JSON.stringify([otherList, otherSingle])).not.toContain(
"secret-plans",
);
expect(otherSingle.action.config).toMatchObject({
channelId: "G_PRIVATE",
channelName: "Private Channel (…ATE)",
});
// Saving the masked config unchanged keeps the real channel.
await otherUserCaller.automations.updateAutomation({
...slackAutomationInput(project.id, "G_PRIVATE"),
automationId: created.automation.id,
name: "Renamed",
actionConfig: {
...(otherSingle.action.config as { channelId: string }),
type: "SLACK",
channelName: "Private Channel (…ATE)",
},
});
const installerView = await caller.automations.getAutomation({
projectId: project.id,
automationId: created.automation.id,
});
expect(installerView.action.config).toMatchObject({
channelName: "secret-plans",
isPrivate: true,
});
// The installer loses the name once they leave the channel.
fakeSlack.installerChannelIds = [];
const afterLeaving = await caller.automations.getAutomation({
projectId: project.id,
automationId: created.automation.id,
});
expect(afterLeaving.action.config).toMatchObject({
channelName: "Private Channel (…ATE)",
});
});
});
describe("automations.updateAutomation", () => {
it("should update an existing automation", async () => {
const { project, caller } = await prepare();
// Create initial automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Original Name",
},
});
const response = await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Updated Name",
eventSource: "prompt",
eventAction: ["created", "updated"],
filter: [],
status: JobConfigState.INACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
"X-Custom": { secret: false, value: "value" },
},
apiVersion: { prompt: "v1" },
},
});
expect(response.trigger).toMatchObject({
id: trigger.id,
eventActions: ["created", "updated"],
status: JobConfigState.INACTIVE,
});
expect(response.action.id).toBe(action.id);
expect(response.action.type).toBe("WEBHOOK");
expect(response.action.config).toMatchObject({
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
displayHeaders: {
"Content-Type": { secret: false, value: "application/json" },
"X-Custom": { secret: false, value: "value" },
},
});
// Verify secrets fields are not present in response
expect(response.action.config).not.toHaveProperty("headers");
expect(response.action.config).not.toHaveProperty("secretKey");
// Verify the automation name was updated
const updatedAutomation = await prisma.automation.findFirst({
where: {
id: automation.id,
},
});
expect(updatedAutomation?.name).toBe("Updated Name");
});
it("should update automation with both plain and secret headers", async () => {
const { project, caller } = await prepare();
// Create initial automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-old-header": { secret: false, value: "old-value" },
"x-case-key": { secret: false, value: "some-value" },
},
displayHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-old-header": { secret: false, value: "old-value" },
"x-case-key": { secret: false, value: "some-value" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Headers Update Test",
},
});
const response = await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Updated Headers Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
requestHeaders: {
"content-type": { secret: false, value: "" },
"x-public": { secret: false, value: "new-public-value" },
"x-secret-key": { secret: true, value: "new-secret-123" },
"x-Case-KEY": { secret: false, value: "new-value" },
},
apiVersion: { prompt: "v1" },
},
});
const actionConfig = response.action.config as SafeWebhookActionConfig;
// Verify the API response contains safe display values
expect(actionConfig.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" }, // header preserved
"x-public": { secret: false, value: "new-public-value" }, // new public header
"x-secret-key": { secret: true, value: "new-...-123" }, // new secret header
"x-Case-KEY": { secret: false, value: "new-value" }, // matched existing key, but new value
});
expect(actionConfig.displayHeaders).not.toHaveProperty(
"x-old-header", // header deleted
);
expect(actionConfig.displayHeaders).not.toHaveProperty(
"x-case-key", // new case replaced the old header name
);
expect(response.action.config).not.toHaveProperty("headers");
expect(response.action.config).not.toHaveProperty("secretKey");
// Verify the action was updated correctly in the database
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const config = updatedAction?.config as any;
// Public headers should remain plain
expect(config.requestHeaders["content-type"].value).toBe(
"application/json",
);
expect(config.requestHeaders["x-public"].value).toBe("new-public-value");
// Secret headers should be encrypted
expect(config.requestHeaders["x-secret-key"].value).not.toBe(
"new-secret-123",
);
expect(config.requestHeaders["x-Case-KEY"].value).toBe("new-value");
expect(config.requestHeaders["x-case-key"]).toBeUndefined();
// Display values should be present with masked secrets
expect(config.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-public": { secret: false, value: "new-public-value" },
"x-secret-key": { secret: true, value: "new-...-123" },
"x-Case-KEY": { secret: false, value: "new-value" },
});
expect(config.displayHeaders).not.toHaveProperty("x-case-key");
});
it("should handle switching header types from secret to plain and vice versa", async () => {
const { project, caller } = await prepare();
// Create initial automation with mixed headers
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-currently-public": { secret: false, value: "public-value" },
"x-currently-secret": { secret: true, value: "secret-value" },
},
displayHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-currently-public": { secret: false, value: "public-value" },
"x-currently-secret": { secret: true, value: "secr...alue" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Header Type Switch Test",
},
});
// Update: switch the header types
const response = await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Switched Headers Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-currently-public": { secret: true, value: "now-secret-value" }, // Was public, now secret
"x-currently-secret": { secret: false, value: "now-public-value" }, // Was secret, now public
},
apiVersion: { prompt: "v1" },
},
});
const actionConfig = response.action.config as SafeWebhookActionConfig;
// Verify the API response contains safe display values reflecting the switch
expect(actionConfig.displayHeaders).toMatchObject({
"x-currently-public": { secret: true, value: "now-...alue" },
"x-currently-secret": { secret: false, value: "now-public-value" },
});
expect(response.action.config).not.toHaveProperty("headers");
expect(response.action.config).not.toHaveProperty("secretKey");
// Verify the switch worked correctly
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const config = updatedAction?.config as WebhookActionConfigWithSecrets;
// x-currently-public should now be encrypted (was plain, now secret)
expect(config.requestHeaders!["x-currently-public"].value).not.toBe(
"now-secret-value",
);
// x-currently-secret should now be plain (was secret, now public)
expect(config.requestHeaders!["x-currently-secret"].value).toBe(
"now-public-value",
);
// Display values should reflect the changes
expect(config.displayHeaders).toMatchObject({
"x-currently-public": { secret: true, value: "now-...alue" },
"x-currently-secret": { secret: false, value: "now-public-value" },
});
});
it("should fail to flip secret header without providing a value", async () => {
const { project, caller } = await prepare();
// Create initial automation with mixed headers
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"x-currently-secret": {
secret: true,
value: encrypt("secret-value"),
},
},
displayHeaders: {
"x-currently-secret": { secret: true, value: "secr...alue" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Header Type Switch Failed Test",
},
});
// Update: try to switch the header types
try {
await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Switched Headers Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
requestHeaders: {
"x-currently-secret": { secret: false, value: "" }, // Was secret, now public
},
apiVersion: { prompt: "v1" },
},
});
throw new Error("Expected an error to be thrown");
} catch (error: any) {
expect(error).toBeInstanceOf(TRPCError);
expect(error.message).toBe(
'Header "x-currently-secret" secret status can only be changed when providing a value',
);
expect(error.code).toBe("BAD_REQUEST");
}
// Verify the switch did not work
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const config = updatedAction?.config as WebhookActionConfigWithSecrets;
// x-currently-secret should still be encrypted
expect(config.requestHeaders!["x-currently-secret"].value).not.toBe(
"secret-value",
);
// Display values should reflect the changes
expect(config.displayHeaders).toMatchObject({
"x-currently-secret": { secret: true, value: "secr...alue" },
});
});
describe("webhook URL changes", () => {
const createWebhookAutomation = async (projectId: string) => {
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: encrypt("secret-key-123") },
},
displayHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "secr...-123" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId,
triggerId: trigger.id,
actionId: action.id,
name: "URL Update Test",
},
});
return { action, automation };
};
const updateWebhook = (
caller: Awaited<ReturnType<typeof prepare>>["caller"],
projectId: string,
automationId: string,
actionConfig: {
url: string;
requestHeaders: Record<string, { secret: boolean; value: string }>;
},
) =>
caller.automations.updateAutomation({
projectId,
automationId,
name: "Updated URL Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
apiVersion: { prompt: "v1" },
...actionConfig,
},
});
it("should reject a URL change when stored secret headers are left blank", async () => {
const { project, caller } = await prepare();
const { action, automation } = await createWebhookAutomation(
project.id,
);
await expect(
updateWebhook(caller, project.id, automation.id, {
url: "https://example.org/webhook",
requestHeaders: {
"content-type": { secret: false, value: "" },
"x-api-key": { secret: true, value: "" },
},
}),
).rejects.toThrow(
'Secret header "x-api-key" must be re-entered when changing the webhook URL',
);
const unchangedAction = await prisma.action.findUniqueOrThrow({
where: { id: action.id },
});
const config = unchangedAction.config as WebhookActionConfigWithSecrets;
expect(config.url).toBe("https://example.com/webhook");
expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(
"secret-key-123",
);
});
it("should drop stored secret headers on a URL change when no headers are submitted", async () => {
const { project, caller } = await prepare();
const { action, automation } = await createWebhookAutomation(
project.id,
);
await updateWebhook(caller, project.id, automation.id, {
url: "https://example.org/webhook",
requestHeaders: {},
});
const updatedAction = await prisma.action.findUniqueOrThrow({
where: { id: action.id },
});
const config = updatedAction.config as WebhookActionConfigWithSecrets;
expect(config.url).toBe("https://example.org/webhook");
expect(config.requestHeaders).toEqual({
"content-type": { secret: false, value: "application/json" },
});
});
it("should accept a URL change when secret headers are re-entered", async () => {
const { project, caller } = await prepare();
const { action, automation } = await createWebhookAutomation(
project.id,
);
await updateWebhook(caller, project.id, automation.id, {
url: "https://example.com/new-webhook-url",
requestHeaders: {
"content-type": { secret: false, value: "" },
"x-api-key": { secret: true, value: "rotated-key-456" },
},
});
const updatedAction = await prisma.action.findUniqueOrThrow({
where: { id: action.id },
});
const config = updatedAction.config as WebhookActionConfigWithSecrets;
expect(config.url).toBe("https://example.com/new-webhook-url");
expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(
"rotated-key-456",
);
expect(config.requestHeaders!["content-type"]).toEqual({
secret: false,
value: "application/json",
});
});
it("should keep stored secret headers when the URL is unchanged", async () => {
const { project, caller } = await prepare();
const { action, automation } = await createWebhookAutomation(
project.id,
);
await updateWebhook(caller, project.id, automation.id, {
url: "https://EXAMPLE.com/webhook",
requestHeaders: {},
});
const updatedAction = await prisma.action.findUniqueOrThrow({
where: { id: action.id },
});
const config = updatedAction.config as WebhookActionConfigWithSecrets;
expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(
"secret-key-123",
);
});
});
it("should migrate legacy headers to new format on update", async () => {
const { project, caller } = await prepare();
// Create initial automation with legacy headers format
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
// Legacy headers format - plain object with string values
headers: {
"content-type": "application/json",
"x-api-key": "legacy-api-key",
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Legacy Headers Migration Test",
},
});
// Update the automation with new headers format, making one secret
const response = await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Migrated Headers Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/updated-webhook",
requestHeaders: {
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "new-secret-key" },
},
apiVersion: { prompt: "v1" },
},
});
const actionConfig = response.action.config as SafeWebhookActionConfig;
// Verify the API response
expect(actionConfig.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "new-...-key" },
});
// Verify the action was updated correctly in the database
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const config = updatedAction?.config as WebhookActionConfigWithSecrets;
// Legacy headers object should be empty
expect(config.headers).toEqual({});
// Secret header should be encrypted in requestHeaders
expect(config.requestHeaders!["x-api-key"].secret).toBe(true);
expect(config.requestHeaders!["x-api-key"].value).not.toBe(
"new-secret-key",
);
// Public header should remain plain
expect(config.requestHeaders!["content-type"].secret).toBe(false);
expect(config.requestHeaders!["content-type"].value).toBe(
"application/json",
);
// Both headers should be in displayHeaders
expect(config.displayHeaders).toMatchObject({
"content-type": { secret: false, value: "application/json" },
"x-api-key": { secret: true, value: "new-...-key" },
});
});
it.each([
{
name: "kept with an empty value",
requestHeaders: { "x-api-key": { secret: true, value: "" } },
},
{ name: "omitted (URL-only update)", requestHeaders: undefined },
])(
"should encrypt a legacy header $name on update",
async ({ requestHeaders }) => {
const { project, caller } = await prepare();
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
headers: { "x-api-key": "legacy-api-key" },
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Legacy Header Encryption Test",
},
});
await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Legacy Header Encryption Test",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "WEBHOOK",
actionConfig: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders,
apiVersion: { prompt: "v1" },
},
});
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const config = updatedAction?.config as WebhookActionConfigWithSecrets;
expect(config.headers).toEqual({});
expect(config.requestHeaders!["x-api-key"].secret).toBe(true);
expect(decrypt(config.requestHeaders!["x-api-key"].value)).toBe(
"legacy-api-key",
);
expect(config.displayHeaders).toEqual({
"x-api-key": { secret: true, value: "lega...-key" },
});
},
);
});
describe("automations.deleteAutomation", () => {
it("should delete an automation and all related data", async () => {
const { project, caller } = await prepare();
// Create automation with executions
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "To Be Deleted",
},
});
// Create some executions
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.COMPLETED,
sourceId: v4(),
input: { test: "data" },
},
});
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { test: "data" },
error: "Test error",
},
});
// Verify data exists before deletion
const beforeTrigger = await prisma.trigger.findUnique({
where: { id: trigger.id },
});
const beforeAction = await prisma.action.findUnique({
where: { id: action.id },
});
const beforeExecutions = await prisma.automationExecution.findMany({
where: { triggerId: trigger.id, actionId: action.id },
});
const beforeAutomation = await prisma.automation.findFirst({
where: { id: automation.id },
});
expect(beforeTrigger).not.toBeNull();
expect(beforeAction).not.toBeNull();
expect(beforeExecutions).toHaveLength(2);
expect(beforeAutomation).not.toBeNull();
// Delete the automation
await caller.automations.deleteAutomation({
projectId: project.id,
automationId: automation.id,
});
// Verify all data is deleted
const afterTrigger = await prisma.trigger.findUnique({
where: { id: trigger.id },
});
const afterAction = await prisma.action.findUnique({
where: { id: action.id },
});
const afterExecutions = await prisma.automationExecution.findMany({
where: { triggerId: trigger.id, actionId: action.id },
});
const afterAutomation = await prisma.automation.findFirst({
where: { id: automation.id },
});
expect(afterTrigger).toBeNull();
expect(afterAction).toBeNull();
expect(afterExecutions).toHaveLength(0);
expect(afterAutomation).toBeNull();
});
});
describe("automations.getAutomationExecutions", () => {
it("should retrieve execution history with pagination", async () => {
const { project, caller } = await prepare();
// Create automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
// Create multiple executions
const executions = [];
for (let i = 0; i < 5; i++) {
const execution = await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status:
i % 2 === 0
? ActionExecutionStatus.COMPLETED
: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: i },
output: i % 2 === 0 ? { result: "success" } : undefined,
error: i % 2 === 1 ? `Error ${i}` : null,
},
});
executions.push(execution);
}
const response = await caller.automations.getAutomationExecutions({
projectId: project.id,
automationId: automation.id,
page: 0,
limit: 3,
});
expect(response.executions).toHaveLength(3);
expect(response.totalCount).toBe(5);
expect(response.executions[0].createdAt.getTime()).toBeGreaterThanOrEqual(
response.executions[1].createdAt.getTime(),
); // Should be ordered by createdAt desc
});
it("should handle pagination correctly", async () => {
const { project, caller } = await prepare();
// Create automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
// Create 10 executions
for (let i = 0; i < 10; i++) {
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.COMPLETED,
sourceId: v4(),
input: { iteration: i },
},
});
}
// Test first page
const page1 = await caller.automations.getAutomationExecutions({
projectId: project.id,
automationId: automation.id,
page: 0,
limit: 5,
});
expect(page1.executions).toHaveLength(5);
expect(page1.totalCount).toBe(10);
// Test second page
const page2 = await caller.automations.getAutomationExecutions({
projectId: project.id,
automationId: automation.id,
page: 1,
limit: 5,
});
expect(page2.executions).toHaveLength(5);
expect(page2.totalCount).toBe(10);
// Verify different executions
const page1Ids = page1.executions.map((e) => e.id);
const page2Ids = page2.executions.map((e) => e.id);
expect(page1Ids).not.toEqual(page2Ids);
});
});
describe("automations.getCountOfConsecutiveFailures", () => {
it("should return consecutive failure count", async () => {
const { project, caller } = await prepare();
// Create automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.INACTIVE, // Disabled due to failures
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
// Create consecutive failed executions
for (let i = 0; i < 3; i++) {
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: i },
error: `Failure ${i}`,
createdAt: new Date(Date.now() - (3 - i) * 60000), // Space them out
},
});
}
const response = await caller.automations.getCountOfConsecutiveFailures({
projectId: project.id,
automationId: automation.id,
});
expect(response.count).toBe(3);
});
it("should return 0 consecutive failures when lastFailingExecutionId is set", async () => {
const { project, caller } = await prepare();
// Create automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.INACTIVE, // Disabled due to failures
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
lastFailingExecutionId: "some-failing-execution-id", // This simulates a webhook that was disabled
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
// Create failed executions that occurred BEFORE the lastFailingExecutionId
await prisma.automationExecution.create({
data: {
id: "some-failing-execution-id",
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: 0 },
error: "Old failure",
createdAt: new Date(Date.now() - 60000), // 1 minute ago
},
});
// Create more failed executions that occurred BEFORE the lastFailingExecutionId
for (let i = 1; i < 5; i++) {
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: i },
error: `Old failure ${i}`,
createdAt: new Date(Date.now() - (60000 + i * 1000)), // Before the lastFailingExecutionId
},
});
}
const response = await caller.automations.getCountOfConsecutiveFailures({
projectId: project.id,
automationId: automation.id,
});
// Should return 0 because all failures occurred before the lastFailingExecutionId
expect(response.count).toBe(0);
});
it("should count failures after lastFailingExecutionId correctly", async () => {
const { project, caller } = await prepare();
// Create automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const { secretKey, displaySecretKey } = generateWebhookSecret();
const lastFailingExecutionId = v4();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
lastFailingExecutionId,
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Test Automation",
},
});
// Create the lastFailingExecution
await prisma.automationExecution.create({
data: {
id: lastFailingExecutionId,
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: 0 },
error: "Last failing execution",
createdAt: new Date(Date.now() - 60000), // 1 minute ago
},
});
// Create old failures that should be ignored
for (let i = 0; i < 3; i++) {
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: i },
error: `Old failure ${i}`,
createdAt: new Date(Date.now() - (120000 + i * 1000)), // Before the lastFailingExecutionId
},
});
}
// Create new failures AFTER the lastFailingExecutionId
for (let i = 0; i < 2; i++) {
await prisma.automationExecution.create({
data: {
id: v4(),
automationId: automation.id,
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { iteration: i },
error: `New failure ${i}`,
createdAt: new Date(Date.now() - (30000 - i * 1000)), // After the lastFailingExecutionId
},
});
}
const response = await caller.automations.getCountOfConsecutiveFailures({
projectId: project.id,
automationId: automation.id,
});
// Should return 2 because only the 2 new failures after lastFailingExecutionId should be counted
expect(response.count).toBe(2);
});
});
describe("automations.regenerateWebhookSecret", () => {
it("should regenerate webhook secret for existing action", async () => {
const { project, caller } = await prepare();
// Create webhook action
const {
secretKey: originalSecretKey,
displaySecretKey: originalDisplaySecretKey,
} = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(originalSecretKey),
displaySecretKey: originalDisplaySecretKey,
},
},
});
const response = await caller.automations.regenerateWebhookSecret({
projectId: project.id,
actionId: action.id,
});
expect(response.webhookSecret).toBeDefined();
expect(response.displaySecretKey).toBeDefined();
expect(response.webhookSecret).not.toBe(originalSecretKey);
expect(response.displaySecretKey).not.toBe(originalDisplaySecretKey);
expect(response.displaySecretKey).toMatch(/^lf-whsec_\.\.\.[a-f0-9]{4}$/);
// Verify the action was updated in the database
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
expect(updatedAction?.config).toMatchObject({
displaySecretKey: response.displaySecretKey,
});
// Critical security test: Verify secret is encrypted in database
const storedSecretKey = (updatedAction?.config as any)?.secretKey;
expect(storedSecretKey).toBeDefined();
expect(storedSecretKey).not.toBe(response.webhookSecret); // Should NOT be plain text
expect(storedSecretKey).not.toBe(originalSecretKey); // Should be different from original
// Verify that the stored secret can be decrypted to match the returned secret
const decryptedStoredSecret = decrypt(storedSecretKey);
expect(decryptedStoredSecret).toBe(response.webhookSecret);
});
it("should preserve custom request headers when regenerating the secret", async () => {
const { project, caller } = await prepare();
const { secretKey, displaySecretKey } = generateWebhookSecret();
const encryptedAuthHeader = encrypt("Bearer super-secret-token");
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
headers: { "X-Legacy": "legacy-value" },
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
Authorization: { secret: true, value: encryptedAuthHeader },
},
displayHeaders: {
"X-Legacy": { secret: false, value: "legacy-value" },
"Content-Type": { secret: false, value: "application/json" },
Authorization: { secret: true, value: "Bear...oken" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
await caller.automations.regenerateWebhookSecret({
projectId: project.id,
actionId: action.id,
});
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const updatedConfig =
updatedAction?.config as WebhookActionConfigWithSecrets;
// Rotating the signing secret must not drop the custom headers, and
// secret header values must stay encrypted at rest.
expect(updatedConfig.requestHeaders).toEqual({
"Content-Type": { secret: false, value: "application/json" },
Authorization: { secret: true, value: encryptedAuthHeader },
});
expect(updatedConfig.headers).toEqual({ "X-Legacy": "legacy-value" });
expect(decrypt(updatedConfig.requestHeaders!.Authorization.value)).toBe(
"Bearer super-secret-token",
);
});
it("should throw error when action not found", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.regenerateWebhookSecret({
projectId: project.id,
actionId: "non-existent-action",
}),
).rejects.toThrow(`Action with id non-existent-action not found.`);
});
it("should handle missing webhook action gracefully", async () => {
const { project, caller } = await prepare();
// Test with non-existent action ID - this will trigger the validation
const actionId = v4();
await expect(
caller.automations.regenerateWebhookSecret({
projectId: project.id,
actionId,
}),
).rejects.toThrow(`Action with id ${actionId} not found.`);
});
it("should throw error when user lacks automations:CUD access", async () => {
const { project, session } = await prepare();
// Create webhook action
const { secretKey, displaySecretKey } = generateWebhookSecret();
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
requestHeaders: {
"Content-Type": { secret: false, value: "application/json" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
// Create a session with limited permissions
const limitedSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: "VIEWER", // VIEWER role doesn't have automations:CUD scope
},
],
},
],
},
};
const limitedCtx = createInnerTRPCContext({
session: limitedSession,
headers: {},
});
const limitedCaller = appRouter.createCaller({ ...limitedCtx, prisma });
await expect(
limitedCaller.automations.regenerateWebhookSecret({
projectId: project.id,
actionId: action.id,
}),
).rejects.toThrow("User does not have access to this resource or action");
});
});
describe("automations.createAutomation with GITHUB_DISPATCH", () => {
it("should create a new GitHub dispatch automation", async () => {
const { project, caller } = await prepare();
// Create test prompt
await prisma.prompt.create({
data: {
id: v4(),
projectId: project.id,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello world" },
createdBy: "test-user",
},
});
const response = await caller.automations.createAutomation({
projectId: project.id,
name: "New GitHub Dispatch Automation",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "langfuse-prompt-created",
githubToken: "ghp_test_token_123456",
},
});
expect(response.trigger).toMatchObject({
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
status: JobConfigState.ACTIVE,
});
expect(response.action).toMatchObject({
projectId: project.id,
type: "GITHUB_DISPATCH",
config: expect.objectContaining({
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "langfuse-prompt-created",
}),
});
expect(response.webhookSecret).toBeDefined();
expect(typeof response.webhookSecret).toBe("string");
expect(response.webhookSecret).toBe("ghp_test_token_123456");
// Verify the automation link was created
const automation = await prisma.automation.findFirst({
where: {
triggerId: response.trigger.id,
actionId: response.action.id,
},
});
expect(automation).toMatchObject({
name: "New GitHub Dispatch Automation",
});
// Verify token is encrypted in database
const createdAction = await prisma.action.findUnique({
where: { id: response.action.id },
});
const config = createdAction?.config as any;
expect(config.githubToken).toBeDefined();
expect(config.githubToken).not.toBe("ghp_test_token_123456"); // Should be encrypted
expect(config.displayGitHubToken).toMatch(/^ghp_...6$/); // Should be masked
// Verify the response doesn't expose the encrypted token
const responseConfig = response.action.config as any;
expect(responseConfig).not.toHaveProperty("githubToken");
expect(responseConfig.displayGitHubToken).toMatch(/^ghp_...6$/);
});
it("should fail to create GitHub dispatch automation with invalid URL", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation({
projectId: project.id,
name: "Invalid GitHub URL",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://example.com/not-github",
eventType: "test-event",
githubToken: "ghp_token_123",
},
}),
).rejects.toThrow();
});
it("should fail to create GitHub dispatch automation without token", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation({
projectId: project.id,
name: "No GitHub Token",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
githubToken: "",
},
}),
).rejects.toThrow();
});
});
describe("automations.updateAutomation with GITHUB_DISPATCH", () => {
it("should reject a GitHub dispatch URL update without a new token", async () => {
const { project, caller } = await prepare();
// Create initial automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "old-event",
githubToken: encrypt("ghp_old_token"),
displayGitHubToken: "ghp_...ken",
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "Original GitHub Dispatch",
},
});
await expect(
caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "Updated GitHub Dispatch",
eventSource: "prompt",
eventAction: ["created", "updated"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/new-repo/dispatches",
eventType: "new-event",
},
}),
).rejects.toThrow(
"GitHub Personal Access Token is required when changing the dispatch URL",
);
const unchangedAction = await prisma.action.findUniqueOrThrow({
where: { id: action.id },
});
const unchangedConfig = unchangedAction.config as any;
expect(unchangedConfig.url).toBe(
"https://api.github.com/repos/owner/repo/dispatches",
);
expect(decrypt(unchangedConfig.githubToken)).toBe("ghp_old_token");
});
it("should update GitHub dispatch automation with new token", async () => {
const { project, caller } = await prepare();
// Create initial automation
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "test-event-type",
githubToken: encrypt("ghp_old_token_123"),
displayGitHubToken: "ghp_...123",
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "GitHub Dispatch Token Update",
},
});
// Update the URL and rotate the token together.
const response = await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "GitHub Dispatch Token Update",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/new-repo/dispatches",
githubToken: "ghp_new_token_456",
},
});
// Verify token is NOT returned (users provided it themselves)
expect(
(response as unknown as { webhookSecret?: string }).webhookSecret,
).toBeUndefined();
const config = response.action.config as any;
expect(config.displayGitHubToken).toMatch(/^ghp_...6$/);
expect(config).not.toHaveProperty("githubToken");
// Verify token is encrypted in database
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const dbConfig = updatedAction?.config as any;
expect(dbConfig.githubToken).not.toBe("ghp_new_token_456"); // Encrypted
expect(decrypt(dbConfig.githubToken)).toBe("ghp_new_token_456"); // Can decrypt
expect(dbConfig.url).toBe(
"https://api.github.com/repos/owner/new-repo/dispatches",
);
});
it("should preserve encrypted token when updating an unchanged URL", async () => {
const { project, caller } = await prepare();
const originalToken = "ghp_original_token_xyz";
const encryptedToken = encrypt(originalToken);
// Create initial automation with encrypted token
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId: project.id,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const action = await prisma.action.create({
data: {
id: v4(),
projectId: project.id,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://github.com/api/v3/repos/owner/repo/dispatches",
eventType: "original-event-type",
githubToken: encryptedToken,
displayGitHubToken: "ghp_...xyz",
},
},
});
const automation = await prisma.automation.create({
data: {
projectId: project.id,
triggerId: trigger.id,
actionId: action.id,
name: "GitHub Dispatch Preserve Token",
},
});
// Update the event type using an equivalent URL representation without
// providing a token. Hostnames are case-insensitive and :443 is the
// default port for HTTPS, so this remains the same destination.
await caller.automations.updateAutomation({
projectId: project.id,
automationId: automation.id,
name: "GitHub Dispatch Preserve Token",
eventSource: "prompt",
eventAction: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "GITHUB_DISPATCH",
actionConfig: {
type: "GITHUB_DISPATCH",
url: "https://GITHUB.COM:443/api/v3/repos/owner/repo/dispatches",
eventType: "new-event-type",
// githubToken intentionally omitted
},
});
// Verify token was preserved (not double-encrypted)
const updatedAction = await prisma.action.findUnique({
where: { id: action.id },
});
const dbConfig = updatedAction?.config as any;
// Token should still be the same encrypted value
expect(dbConfig.githubToken).toBe(encryptedToken);
// Should still decrypt to original plaintext
expect(decrypt(dbConfig.githubToken)).toBe(originalToken);
// Verify URL and eventType were updated
expect(dbConfig.url).toBe(
"https://GITHUB.COM:443/api/v3/repos/owner/repo/dispatches",
);
expect(dbConfig.eventType).toBe("new-event-type");
});
});
describe("automations read path secret redaction", () => {
async function createGitHubDispatchAutomation(
projectId: string,
config: Prisma.InputJsonObject,
) {
const trigger = await prisma.trigger.create({
data: {
id: v4(),
projectId,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
const action = await prisma.action.create({
data: {
id: v4(),
projectId,
type: "GITHUB_DISPATCH",
config,
},
});
return prisma.automation.create({
data: {
projectId,
triggerId: trigger.id,
actionId: action.id,
name: "GitHub Dispatch Read Path",
},
});
}
// VIEWER holds automations:read, the only scope both read routes require.
function viewerCaller(session: Session) {
const viewerSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: "VIEWER",
},
],
},
],
},
};
return appRouter.createCaller({
...createInnerTRPCContext({ session: viewerSession, headers: {} }),
prisma,
});
}
it("should not expose the stored githubToken to a project VIEWER", async () => {
const { project, session } = await prepare();
const automation = await createGitHubDispatchAutomation(project.id, {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "langfuse-prompt-created",
githubToken: encrypt("ghp_read_path_token_123"),
displayGitHubToken: "ghp_...123",
});
const caller = viewerCaller(session);
const automations = await caller.automations.getAutomations({
projectId: project.id,
});
expect(automations).toHaveLength(1);
const listedConfig = automations[0].action.config as Record<
string,
unknown
>;
expect(listedConfig).not.toHaveProperty("githubToken");
expect(listedConfig.displayGitHubToken).toBe("ghp_...123");
const single = await caller.automations.getAutomation({
projectId: project.id,
automationId: automation.id,
});
const singleConfig = single.action.config as Record<string, unknown>;
expect(singleConfig).not.toHaveProperty("githubToken");
expect(singleConfig.displayGitHubToken).toBe("ghp_...123");
});
it("should not expose the githubToken of a config that fails to parse", async () => {
const { project, session } = await prepare();
// displayGitHubToken is missing, so the config does not parse as a
// GITHUB_DISPATCH config and only the field allowlist can strip it.
const automation = await createGitHubDispatchAutomation(project.id, {
type: "GITHUB_DISPATCH",
url: "https://api.github.com/repos/owner/repo/dispatches",
eventType: "langfuse-prompt-created",
githubToken: encrypt("ghp_read_path_token_456"),
});
const caller = viewerCaller(session);
const single = await caller.automations.getAutomation({
projectId: project.id,
automationId: automation.id,
});
const config = single.action.config as Record<string, unknown>;
expect(config).not.toHaveProperty("githubToken");
expect(config.type).toBe("GITHUB_DISPATCH");
expect(config.eventType).toBe("langfuse-prompt-created");
});
});
describe("PagerDuty actions", () => {
const routingKey = "R0abcdefghijklmnopqrstuvwxyz9xYz";
const newRoutingKey = "N3wkeyabcdefghijklmnopqrstuv7uVw";
const createInput = (
projectId: string,
actionConfig: {
routingKey?: string;
region: "US" | "EU";
} = { routingKey, region: "US" },
) => ({
projectId,
name: "Page on-call",
eventSource: TriggerEventSource.Monitor,
eventAction: [],
filter: [],
status: JobConfigState.ACTIVE,
actionType: "PAGERDUTY" as const,
actionConfig: { type: "PAGERDUTY" as const, ...actionConfig },
});
async function storedConfig(actionId: string) {
const action = await prisma.action.findUniqueOrThrow({
where: { id: actionId },
});
return action.config as Record<string, string>;
}
function viewerCaller(session: Session) {
const viewerSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: "VIEWER",
},
],
},
],
},
};
return appRouter.createCaller({
...createInnerTRPCContext({ session: viewerSession, headers: {} }),
prisma,
});
}
it("encrypts the routing key and returns only the masked key", async () => {
const { project, caller } = await prepare();
const result = await caller.automations.createAutomation(
createInput(project.id, { routingKey, region: "EU" }),
);
const config = await storedConfig(result.action.id);
expect(config.routingKey).not.toBe(routingKey);
expect(decrypt(config.routingKey)).toBe(routingKey);
expect(config.displayRoutingKey).toBe("R0ab...9xYz");
expect(config.region).toBe("EU");
expect(result.action.config).toEqual({
type: "PAGERDUTY",
displayRoutingKey: "R0ab...9xYz",
region: "EU",
});
});
function callerWithUser(session: Session, user: Partial<Session["user"]>) {
return appRouter.createCaller({
...createInnerTRPCContext({
session: { ...session, user: { ...session.user!, ...user } },
headers: {},
}),
prisma,
});
}
it("lets project owners who are not admins create PagerDuty automations", async () => {
const { project, session } = await prepare();
const owner = callerWithUser(session, { admin: false });
const result = await owner.automations.createAutomation(
createInput(project.id),
);
expect(result.action.type).toBe("PAGERDUTY");
});
it("rejects a create without a routing key", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation(
createInput(project.id, { region: "US" }),
),
).rejects.toMatchObject({ code: "BAD_REQUEST" });
});
it("rejects a malformed routing key", async () => {
const { project, caller } = await prepare();
await expect(
caller.automations.createAutomation(
createInput(project.id, { routingKey: "too-short", region: "US" }),
),
).rejects.toMatchObject({ code: "BAD_REQUEST" });
});
it("keeps the stored key when an update omits it", async () => {
const { project, caller } = await prepare();
const created = await caller.automations.createAutomation(
createInput(project.id),
);
await caller.automations.updateAutomation({
...createInput(project.id, { region: "EU" }),
automationId: created.automation.id,
});
const config = await storedConfig(created.action.id);
expect(decrypt(config.routingKey)).toBe(routingKey);
expect(config.displayRoutingKey).toBe("R0ab...9xYz");
expect(config.region).toBe("EU");
});
it("replaces the key when an update sends a new one", async () => {
const { project, caller } = await prepare();
const created = await caller.automations.createAutomation(
createInput(project.id),
);
await caller.automations.updateAutomation({
...createInput(project.id, {
routingKey: newRoutingKey,
region: "US",
}),
automationId: created.automation.id,
});
const config = await storedConfig(created.action.id);
expect(decrypt(config.routingKey)).toBe(newRoutingKey);
expect(config.displayRoutingKey).toBe("N3wk...7uVw");
});
it("never returns the routing key on any read path to a project VIEWER", async () => {
const { project, session, caller } = await prepare();
const created = await caller.automations.createAutomation(
createInput(project.id),
);
const storedKey = (await storedConfig(created.action.id)).routingKey;
const viewer = viewerCaller(session);
const responses = [
await viewer.automations.getAutomations({ projectId: project.id }),
await viewer.automations.getAutomation({
projectId: project.id,
automationId: created.automation.id,
}),
];
for (const response of responses) {
const serialized = JSON.stringify(response);
expect(serialized).not.toContain(routingKey);
expect(serialized).not.toContain(storedKey);
expect(serialized).not.toContain("routingKey");
}
});
describe("sendPagerDutyTestEvent", () => {
let fetchSpy: MockInstance<typeof fetch>;
beforeEach(() => {
fetchSpy = vi
.spyOn(global, "fetch")
.mockResolvedValue(new Response("{}", { status: 202 }));
});
afterEach(() => {
fetchSpy.mockRestore();
});
function sentEvents() {
return fetchSpy.mock.calls.map(([url, init]) => ({
url: String(url),
body: JSON.parse(String((init as RequestInit).body)),
}));
}
it.each([
["trigger", "trigger", "critical"],
["resolve", "resolve", undefined],
] as const)(
"sends the %s step as one %s under the project's test key",
async (step, eventAction, severity) => {
const { project, caller } = await prepare();
await caller.automations.sendPagerDutyTestEvent({
projectId: project.id,
region: "EU",
routingKey,
step,
});
const [event, ...rest] = sentEvents();
expect(rest).toEqual([]);
expect(event.url).toBe("https://events.eu.pagerduty.com/v2/enqueue");
expect(event.body).toMatchObject({
routing_key: routingKey,
event_action: eventAction,
dedup_key: `langfuse-${project.id}-test`,
});
expect(event.body.payload?.severity).toBe(severity);
},
);
it("falls back to the stored key of the action", async () => {
const { project, caller } = await prepare();
const created = await caller.automations.createAutomation(
createInput(project.id),
);
await caller.automations.sendPagerDutyTestEvent({
projectId: project.id,
region: "US",
actionId: created.action.id,
step: "trigger",
});
expect(sentEvents()[0]).toMatchObject({
url: "https://events.pagerduty.com/v2/enqueue",
body: { routing_key: routingKey },
});
});
it("rejects a stored key of another project", async () => {
const { project: otherProject, caller: otherCaller } = await prepare();
const otherAction = await otherCaller.automations.createAutomation(
createInput(otherProject.id),
);
const { project, caller } = await prepare();
await expect(
caller.automations.sendPagerDutyTestEvent({
projectId: project.id,
region: "US",
actionId: otherAction.action.id,
step: "trigger",
}),
).rejects.toMatchObject({ code: "BAD_REQUEST" });
expect(fetchSpy).not.toHaveBeenCalled();
});
it("maps a PagerDuty rejection to BAD_REQUEST", async () => {
fetchSpy.mockResolvedValue(
new Response("invalid key", { status: 400 }),
);
const { project, caller } = await prepare();
await expect(
caller.automations.sendPagerDutyTestEvent({
projectId: project.id,
region: "US",
routingKey,
step: "trigger",
}),
).rejects.toMatchObject({ code: "BAD_REQUEST" });
});
it.each([
[429, "TOO_MANY_REQUESTS"],
[502, "BAD_GATEWAY"],
[503, "SERVICE_UNAVAILABLE"],
])("returns a PagerDuty %i as %s after retries", async (status, code) => {
fetchSpy.mockResolvedValue(new Response("busy", { status }));
const { project, caller } = await prepare();
await expect(
caller.automations.sendPagerDutyTestEvent({
projectId: project.id,
region: "US",
routingKey,
step: "trigger",
}),
).rejects.toMatchObject({ code });
expect(fetchSpy).toHaveBeenCalledTimes(4);
});
});
});
});