🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { LLMAdapter, type Plan, Role } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
createApiKey,
createApiKeyCacheKey,
AUTHZ_CONTEXT_CACHE_KEY_PREFIX,
createAuthzContextCacheKey,
createShaHash,
} from "@langfuse/shared/src/server";
import {
OrganizationId,
ProjectId,
SystemRoleId,
UserId,
} from "@langfuse/shared/rbac";
import { assignRole } from "@langfuse/shared/rbac/server";
import type { Session } from "next-auth";
import { v4 as uuidv4 } from "uuid";
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from "vitest";
const mocks = vi.hoisted(() => ({
reconcileTopicsConnectionsForProject: vi.fn(),
}));
vi.mock("@langfuse/shared/topics/server", () => ({
reconcileTopicsConnectionsForProject:
mocks.reconcileTopicsConnectionsForProject,
}));
// organizations.delete cancels Stripe before deleting; the test env has a cloud
// region but no Stripe, so force the self-hosted path to reach the eviction.
vi.mock("@/src/ee/features/billing/utils/isCloudBillingEnabled", () => ({
isCloudBillingEnabled: () => false,
}));
vi.mock(
"@/src/ee/features/billing/utils/isCloudBillingEnabled",
async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
return {
...actual,
isCloudBillingEnabled: () => false,
};
},
);
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { env } from "@/src/env.mjs";
import { handleDeleteProject } from "@/src/ee/features/admin-api/server/projects/projectById";
import { handleDeleteOrganization } from "@/src/ee/features/admin-api/server/organizations/organizationById";
import {
createRedisTestClient,
ensureRedisReady,
getRedisKeysByPattern,
setRedisValue,
type RedisTestClient,
} from "@/src/__tests__/server/redis-test-utils";
let redis: RedisTestClient;
beforeAll(async () => {
redis = createRedisTestClient();
await ensureRedisReady(redis);
});
afterAll(() => {
redis.disconnect();
});
async function createOrg() {
const orgId = uuidv4();
await prisma.organization.create({
data: { id: orgId, name: `Org ${orgId.slice(0, 8)}` },
});
return orgId;
}
async function createProject(orgId: string) {
const projectId = uuidv4();
await prisma.project.create({
data: { id: projectId, name: `Project ${projectId.slice(0, 8)}`, orgId },
});
return projectId;
}
function cacheKeysFor({
fastHash,
publicKey,
}: {
fastHash: string;
publicKey: string;
}) {
return [
createApiKeyCacheKey(fastHash),
`${AUTHZ_CONTEXT_CACHE_KEY_PREFIX}${fastHash}`,
`${AUTHZ_CONTEXT_CACHE_KEY_PREFIX}${createShaHash(publicKey, env.SALT!)}`,
createAuthzContextCacheKey("basic", fastHash),
createAuthzContextCacheKey("bearer", fastHash),
createAuthzContextCacheKey("bearer", createShaHash(publicKey, env.SALT!)),
];
}
async function seedOrgScopedKey(orgId: string) {
const { publicKey, secretKey } = await createApiKey(prisma, {
tenant: OrganizationId(orgId),
scope: OrganizationId(orgId),
role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
createdBy: "system",
});
const fastHash = createShaHash(secretKey, env.SALT!);
const keys = cacheKeysFor({ fastHash, publicKey });
for (const key of keys) {
await setRedisValue(redis, key, "1");
}
return keys;
}
async function survivingKeys(keys: string[]) {
const present: string[] = [];
for (const key of keys) {
const [hit] = await getRedisKeysByPattern(redis, key);
if (hit) present.push(hit);
}
return present;
}
async function createUserInOrgs(orgIds: string[]) {
const userId = uuidv4();
const user = await prisma.user.create({
data: {
id: userId,
email: `cache-inval-${userId.slice(0, 8)}@test.com`,
name: "Test User",
},
});
for (const orgId of orgIds) {
await prisma.organizationMembership.create({
data: { userId: user.id, orgId, role: Role.OWNER },
});
}
return user;
}
function makeCaller({
userId,
orgIds,
projectId,
plan = "cloud:pro",
}: {
userId: string;
orgIds: string[];
projectId?: string;
plan?: Plan;
}) {
const session: Session = {
expires: "1",
user: {
id: userId,
canCreateOrganizations: true,
name: "Test User",
email: "[email protected]",
organizations: orgIds.map((orgId) => ({
id: orgId,
name: "Test Organization",
role: Role.OWNER,
plan,
cloudConfig: undefined,
metadata: {},
aiFeaturesEnabled: false,
aiTelemetryEnabled: true,
projects: projectId
? [
{
id: projectId,
role: Role.OWNER,
retentionDays: 0,
deletedAt: null,
hasTraces: false,
name: "Test Project",
metadata: {},
createdAt: new Date().toISOString(),
},
]
: [],
})),
featureFlags: testFeatureFlags(),
admin: false,
},
environment: {} as any,
};
const ctx = createInnerTRPCContext({ session, headers: {} });
return appRouter.createCaller({ ...ctx, prisma, session });
}
function makeRes() {
const res: any = {};
res.statusCode = 200;
res.status = (code: number) => {
res.statusCode = code;
return res;
};
res.json = (body: unknown) => {
res.body = body;
return res;
};
return res;
}
describe("API-key cache invalidation on project/org lifecycle", () => {
beforeEach(async () => {
await ensureRedisReady(redis);
});
it("projects.delete evicts the org's cached keys", async () => {
const orgId = await createOrg();
const projectId = await createProject(orgId);
const user = await createUserInOrgs([orgId]);
const keys = await seedOrgScopedKey(orgId);
const caller = makeCaller({ userId: user.id, orgIds: [orgId], projectId });
await caller.projects.delete({ projectId });
expect(await survivingKeys(keys)).toEqual([]);
});
it("projects.transfer evicts cached keys for both source and target orgs", async () => {
const sourceOrgId = await createOrg();
const targetOrgId = await createOrg();
const projectId = await createProject(sourceOrgId);
const user = await createUserInOrgs([sourceOrgId, targetOrgId]);
const sourceKeys = await seedOrgScopedKey(sourceOrgId);
const targetKeys = await seedOrgScopedKey(targetOrgId);
const caller = makeCaller({
userId: user.id,
orgIds: [sourceOrgId, targetOrgId],
projectId,
});
await caller.projects.transfer({ projectId, targetOrgId });
expect(await survivingKeys([...sourceKeys, ...targetKeys])).toEqual([]);
});
it("projects.transfer rebinds the default eval model to the destination organization", async () => {
const sourceOrgId = await createOrg();
const targetOrgId = await createOrg();
const projectId = await createProject(sourceOrgId);
const user = await createUserInOrgs([sourceOrgId, targetOrgId]);
const provider = `transfer-provider-${uuidv4()}`;
const sourceConnection = await prisma.llmApiKeys.create({
data: {
organizationId: sourceOrgId,
provider,
adapter: LLMAdapter.OpenAI,
displaySecretKey: "source",
secretKey: "source",
},
});
const targetConnection = await prisma.llmApiKeys.create({
data: {
organizationId: targetOrgId,
provider,
adapter: LLMAdapter.OpenAI,
displaySecretKey: "target",
secretKey: "target",
},
});
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: sourceConnection.id,
provider,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o-mini",
},
});
const caller = makeCaller({
userId: user.id,
orgIds: [sourceOrgId, targetOrgId],
projectId,
});
await caller.projects.transfer({ projectId, targetOrgId });
await expect(
prisma.defaultLlmModel.findUnique({ where: { projectId } }),
).resolves.toMatchObject({ llmApiKeyId: targetConnection.id });
});
it("projects.transfer removes the default eval model when no compatible connection exists", async () => {
const sourceOrgId = await createOrg();
const targetOrgId = await createOrg();
const projectId = await createProject(sourceOrgId);
const user = await createUserInOrgs([sourceOrgId, targetOrgId]);
const provider = `transfer-provider-${uuidv4()}`;
const sourceConnection = await prisma.llmApiKeys.create({
data: {
organizationId: sourceOrgId,
provider,
adapter: LLMAdapter.OpenAI,
displaySecretKey: "source",
secretKey: "source",
},
});
await prisma.llmApiKeys.create({
data: {
organizationId: targetOrgId,
provider,
adapter: LLMAdapter.Anthropic,
displaySecretKey: "target",
secretKey: "target",
},
});
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: sourceConnection.id,
provider,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o-mini",
},
});
const caller = makeCaller({
userId: user.id,
orgIds: [sourceOrgId, targetOrgId],
projectId,
});
await caller.projects.transfer({ projectId, targetOrgId });
await expect(
prisma.defaultLlmModel.findUnique({ where: { projectId } }),
).resolves.toBeNull();
});
it("projects.transfer still evicts cached keys when Topics refresh fails", async () => {
const sourceOrgId = await createOrg();
const targetOrgId = await createOrg();
const projectId = await createProject(sourceOrgId);
const user = await createUserInOrgs([sourceOrgId, targetOrgId]);
const sourceKeys = await seedOrgScopedKey(sourceOrgId);
const targetKeys = await seedOrgScopedKey(targetOrgId);
mocks.reconcileTopicsConnectionsForProject.mockRejectedValueOnce(
new Error("Topics refresh failed"),
);
const caller = makeCaller({
userId: user.id,
orgIds: [sourceOrgId, targetOrgId],
projectId,
});
await expect(
caller.projects.transfer({ projectId, targetOrgId }),
).rejects.toThrow();
expect(await survivingKeys([...sourceKeys, ...targetKeys])).toEqual([]);
});
it("organizations.delete evicts cached keys before the cascade removes the rows", async () => {
const orgId = await createOrg();
const user = await createUserInOrgs([orgId]);
const keys = await seedOrgScopedKey(orgId);
const caller = makeCaller({ userId: user.id, orgIds: [orgId] });
await caller.organizations.delete({ orgId });
expect(await survivingKeys(keys)).toEqual([]);
});
it("accessAuditLog.setEnabled evicts the org's cached keys", async () => {
const orgId = await createOrg();
const user = await createUserInOrgs([orgId]);
const keys = await seedOrgScopedKey(orgId);
const caller = makeCaller({
userId: user.id,
orgIds: [orgId],
plan: "cloud:enterprise",
});
await caller.accessAuditLog.setEnabled({ orgId, enabled: true });
expect(await survivingKeys(keys)).toEqual([]);
});
it("admin project soft deletion preserves user permissions and evicts cached keys", async () => {
const orgId = await createOrg();
const projectId = await createProject(orgId);
const keys = await seedOrgScopedKey(orgId);
const user = await createUserInOrgs([orgId]);
const assignment = await assignRole(prisma, {
tenantId: OrganizationId(orgId),
ownerId: ProjectId(projectId),
principalId: UserId(user.id),
roleId: SystemRoleId("VIEWER"),
tags: [],
});
const res = makeRes();
await handleDeleteProject({} as any, res, projectId, {
orgId,
apiKeyId: "ADMIN_KEY",
} as any);
expect(res.statusCode).toBe(202);
expect(await survivingKeys(keys)).toEqual([]);
await expect(
prisma.roleAssignment.findUnique({ where: { id: assignment.id } }),
).resolves.toEqual(assignment);
await expect(
prisma.project.findUnique({ where: { id: projectId } }),
).resolves.toMatchObject({ deletedAt: expect.any(Date) });
});
it("admin handleDeleteOrganization evicts cached keys before the cascade removes the rows", async () => {
const orgId = await createOrg();
const keys = await seedOrgScopedKey(orgId);
const res = makeRes();
await handleDeleteOrganization(
{ query: { organizationId: orgId } } as any,
res,
);
expect(res.statusCode).toBe(200);
expect(await survivingKeys(keys)).toEqual([]);
});
});