🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, it, expect, beforeEach } from "vitest";
import { v4 } from "uuid";
import {
ActionExecutionStatus,
JobConfigState,
TriggerEventSource,
PromptType,
} from "@langfuse/shared";
import {
createOrgProjectAndApiKey,
EntityChangeEventType,
EntityChangeEventSchema,
} from "@langfuse/shared/src/server";
import { ActionType, prisma } from "@langfuse/shared/src/db";
import { skillVersionProcessor } from "../features/entityChange/skillVersionProcessor";
import { promptVersionProcessor } from "../features/entityChange/promptVersionProcessor";
describe("promptVersionChangeWorker", () => {
let projectId: string;
beforeEach(async () => {
const result = await createOrgProjectAndApiKey();
projectId = result.projectId;
});
it.each([
// eventActions only
{
description: "should execute webhook for matching prompt created event",
trigger: { eventActions: ["created"], nameFilter: null },
event: { promptName: "test-prompt", action: "created" as const },
expected: 1,
},
{
description: "should execute webhook for matching prompt updated event",
trigger: { eventActions: ["updated"], nameFilter: null },
event: { promptName: "test-prompt", action: "updated" as const },
expected: 1,
},
{
description: "should execute webhook for matching prompt deleted event",
trigger: { eventActions: ["deleted"], nameFilter: null },
event: { promptName: "test-prompt", action: "deleted" as const },
expected: 1,
},
{
description: "should not execute when action filter doesn't match",
trigger: { eventActions: ["deleted"], nameFilter: null },
event: { promptName: "test-prompt", action: "created" as const },
expected: 0,
},
{
description:
"should execute webhook when eventActions and filters are empty",
trigger: { eventActions: [] as string[], nameFilter: null },
event: { promptName: "test-prompt", action: "updated" as const },
expected: 1,
},
// eventActions + name filter combinations
{
description:
"should execute webhook when eventActions is empty and name filter matches",
trigger: { eventActions: [] as string[], nameFilter: "target-prompt" },
event: { promptName: "target-prompt", action: "created" as const },
expected: 1,
},
{
description:
"should execute webhook when eventActions and name filter both match",
trigger: { eventActions: ["created"], nameFilter: "target-prompt" },
event: { promptName: "target-prompt", action: "created" as const },
expected: 1,
},
{
description:
"should not execute when eventActions matches but name filter does not",
trigger: { eventActions: ["created"], nameFilter: "target-prompt" },
event: { promptName: "different-prompt", action: "created" as const },
expected: 0,
},
{
description:
"should not execute when eventActions does not match but name filter does",
trigger: { eventActions: ["deleted"], nameFilter: "target-prompt" },
event: { promptName: "target-prompt", action: "created" as const },
expected: 0,
},
])(
"$description",
async ({ trigger, event: { promptName, action }, expected }) => {
const { eventActions, nameFilter } = trigger;
const promptId = v4();
const actionId = v4();
await prisma.action.create({
data: {
id: actionId,
projectId,
type: ActionType.WEBHOOK,
config: {
type: "WEBHOOK",
url: "https://webhook.example.com/test",
headers: {},
method: "POST",
},
},
});
const triggerId = v4();
await prisma.trigger.create({
data: {
id: triggerId,
projectId,
eventSource: TriggerEventSource.Prompt,
eventActions,
status: JobConfigState.ACTIVE,
filter: nameFilter
? [
{
column: "Name",
operator: "=",
value: nameFilter,
type: "string",
},
]
: [],
},
});
const automationId = v4();
await prisma.automation.create({
data: {
id: automationId,
name: `automation-${v4()}`,
projectId,
triggerId,
actionId,
},
});
const event: EntityChangeEventType = {
entityType: "prompt-version",
projectId,
promptId,
action,
prompt: {
id: promptId,
projectId,
name: promptName,
version: 1,
prompt: { messages: [{ role: "user", content: "Hello" }] },
config: null,
tags: [],
labels: [],
type: PromptType.Chat,
isActive: true,
createdBy: "test-user",
createdAt: new Date(),
updatedAt: new Date(),
commitMessage: null,
},
user: {
id: "test-user",
name: "Test User",
email: "[email protected]",
},
};
await promptVersionProcessor(event);
const executions = await prisma.automationExecution.findMany({
where: { projectId, automationId },
});
expect(executions).toHaveLength(expected);
if (expected > 0) {
expect(executions[0].status).toBe(ActionExecutionStatus.PENDING);
expect(executions[0].sourceId).toBe(promptId);
}
},
);
it("routes serialized skill events only to matching skill automations", async () => {
const automationId = v4();
for (const source of [
TriggerEventSource.Skill,
TriggerEventSource.Prompt,
]) {
await prisma.automation.create({
data: {
id: source === TriggerEventSource.Skill ? automationId : v4(),
project: { connect: { id: projectId } },
name: `${source} automation`,
action: {
create: {
projectId,
type: ActionType.WEBHOOK,
config: {
type: "WEBHOOK",
url: "https://webhook.example.com/test",
headers: {},
},
},
},
trigger: {
create: {
projectId,
eventSource: source,
eventActions: ["created"],
status: JobConfigState.ACTIVE,
filter: [
{
column: "Name",
operator: "=",
value: "test-skill",
type: "string",
},
],
},
},
},
});
}
const skillId = v4();
const event: Extract<
EntityChangeEventType,
{ entityType: "skill-version" }
> = {
entityType: "skill-version",
projectId,
skillId,
action: "created",
skill: {
id: skillId,
projectId,
name: "test-skill",
description: "Test skill",
frontmatter: {},
version: 2,
tags: [],
labels: ["latest"],
commitMessage: null,
files: [],
createdBy: "test-user",
createdAt: new Date(),
updatedAt: new Date(),
},
};
const hydrated = EntityChangeEventSchema.parse(
JSON.parse(JSON.stringify(event)),
);
if (hydrated.entityType !== "skill-version")
throw new Error("Unexpected event type");
expect(hydrated.skill.createdAt).toBeInstanceOf(Date);
await skillVersionProcessor(hydrated);
const executions = await prisma.automationExecution.findMany({
where: { projectId },
});
expect(executions).toHaveLength(1);
expect(executions[0]).toMatchObject({
automationId,
sourceId: skillId,
status: ActionExecutionStatus.PENDING,
input: { type: "skill-version", skillVersion: 2 },
});
await skillVersionProcessor({
...event,
skill: { ...event.skill, name: "different-skill" },
});
expect(
await prisma.automationExecution.count({ where: { projectId } }),
).toBe(1);
await expect(
skillVersionProcessor({ ...event, projectId: v4() }),
).rejects.toThrow("identity does not match");
});
});