🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { expect, describe, it, beforeEach } from "vitest";
import { v4 as uuidv4 } from "uuid";
import { prisma } from "@langfuse/shared/src/db";
import PatchLLMToolAndLLLMSchemaAuditLogs from "../backgroundMigrations/patchLLMToolAndLLLMSchemaAuditLogs";
describe("PatchLLMToolAndLLLMSchemaAuditLogs", () => {
let migration: PatchLLMToolAndLLLMSchemaAuditLogs;
let testOrgId: string;
let testProjectId: string;
beforeEach(async () => {
migration = new PatchLLMToolAndLLLMSchemaAuditLogs();
testOrgId = uuidv4();
testProjectId = uuidv4();
});
it("should correctly patch various audit log scenarios", async () => {
// Create test data with different scenarios
const auditLogs = [
// 1. Valid llmTool with parameters in before
{
id: `aa-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should be corrected
resourceId: uuidv4(), // Different from project ID
action: "create",
before: JSON.stringify({
name: "test-tool",
parameters: {
type: "object",
properties: { input: { type: "string" } },
},
description: "Test tool",
}),
after: null,
},
// 2. Valid llmSchema with schema in after
{
id: `ab-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should be corrected
resourceId: uuidv4(), // Different from project ID
action: "update",
before: null,
after: JSON.stringify({
name: "test-schema",
schema: {
type: "object",
properties: { output: { type: "string" } },
},
description: "Test schema",
}),
},
// 3. Valid llmTool with parameters in after (before is null)
{
id: `ac-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should be corrected
resourceId: uuidv4(), // Different from project ID
action: "create",
before: null,
after: JSON.stringify({
name: "tool-after",
parameters: {
type: "object",
properties: { param1: { type: "number" } },
},
description: "Tool with parameters in after",
}),
},
// 4. Valid llmSchema with schema in before
{
id: `ad-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should be corrected
resourceId: uuidv4(), // Different from project ID
action: "delete",
before: JSON.stringify({
name: "schema-before",
schema: { type: "array", items: { type: "string" } },
description: "Schema with schema in before",
}),
after: null,
},
// 5. Empty parameters
{
id: `ae-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should remain project
resourceId: uuidv4(), // Different from project ID
action: "update",
before: JSON.stringify({
name: "empty-params",
parameters: {},
description: "Empty parameters",
}),
after: null,
},
// 6. Empty schema
{
id: `af-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should remain project
resourceId: uuidv4(), // Different from project ID
action: "update",
before: JSON.stringify({
name: "empty-schema",
schema: {},
description: "Empty schema",
}),
after: null,
},
// 7. Record with malformed JSON in before
{
id: `ah-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should remain project due to malformed JSON
resourceId: uuidv4(), // Different from project ID
action: "update",
before: "invalid json {",
after: null,
},
// 8. Record with malformed JSON in after
{
id: `ai-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should remain project due to malformed JSON
resourceId: uuidv4(), // Different from project ID
action: "update",
before: null,
after: "invalid json }",
},
// 9. Record with correct resource type already (should not be affected)
{
id: `aj-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "llmTool", // Already correct
resourceId: uuidv4(), // Different from project ID
action: "update",
before: JSON.stringify({
name: "already-correct",
parameters: {
type: "object",
properties: { input: { type: "string" } },
},
description: "Already correct resource type",
}),
after: null,
},
// 10. Record with resourceId matching projectId (should not be affected)
{
id: `ak-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project",
resourceId: testProjectId, // Same as project ID
action: "update",
before: JSON.stringify({
name: "matching-ids",
parameters: {
type: "object",
properties: { input: { type: "string" } },
},
description: "Resource ID matches project ID",
}),
after: null,
},
// 11. Record with null/empty before and after
{
id: `al-${uuidv4()}`,
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should remain project due to no data
resourceId: uuidv4(), // Different from project ID
action: "update",
before: null,
after: null,
},
];
// Insert test audit logs
await prisma.auditLog.createMany({
data: auditLogs,
});
// Run migration
await migration.run();
// Verify results
const updatedLogs = await prisma.auditLog.findMany({
where: {
id: { in: auditLogs.map((log) => log.id) },
},
orderBy: { id: "asc" },
});
// Check each scenario
expect(updatedLogs[0].resourceType).toBe("llmTool"); // Has parameters in before
expect(updatedLogs[1].resourceType).toBe("llmSchema"); // Has schema in after
expect(updatedLogs[2].resourceType).toBe("llmTool"); // Has parameters in after
expect(updatedLogs[3].resourceType).toBe("llmSchema"); // Has schema in before
expect(updatedLogs[4].resourceType).toBe("llmTool"); // Empty parameters
expect(updatedLogs[5].resourceType).toBe("llmSchema"); // Empty schema
expect(updatedLogs[6].resourceType).toBe("project"); // Malformed JSON in before
expect(updatedLogs[7].resourceType).toBe("project"); // Malformed JSON in after
expect(updatedLogs[8].resourceType).toBe("llmTool"); // Already correct, should not be affected
expect(updatedLogs[9].resourceType).toBe("project"); // Resource ID matches project ID
expect(updatedLogs[10].resourceType).toBe("project"); // Null/empty before and after
// Clean up
await prisma.auditLog.deleteMany({
where: {
id: { in: auditLogs.map((log) => log.id) },
},
});
}, 60_000);
it("should handle batching correctly for large datasets", async () => {
// Create more than 1000 audit logs to test batching
const largeDataset = Array.from({ length: 1250 }, (_, index) => ({
id: uuidv4(),
orgId: testOrgId,
projectId: testProjectId,
resourceType: "project", // Wrong type, should be corrected
resourceId: uuidv4(), // Different from project ID
action: "create",
before: JSON.stringify({
name: `test-tool-${index}`,
parameters: {
type: "object",
properties: { input: { type: "string" } },
},
description: `Test tool ${index}`,
}),
after: null,
}));
// Insert test audit logs
await prisma.auditLog.createMany({
data: largeDataset,
});
// Run migration
await migration.run();
// Verify all records were processed
const updatedLogs = await prisma.auditLog.findMany({
where: {
id: { in: largeDataset.map((log) => log.id) },
},
});
// All should be updated to llmTool
expect(updatedLogs.every((log) => log.resourceType === "llmTool")).toBe(
true,
);
expect(updatedLogs.length).toBe(1250);
// Clean up
await prisma.auditLog.deleteMany({
where: {
id: { in: largeDataset.map((log) => log.id) },
},
});
}, 60_000);
});