🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(audit-logs): omit dataset item payloads from audit logs (#19111)
marliessophiea8da34e054d2
diff --git a/web/src/__tests__/server/audit-log-staging.servertest.ts b/web/src/__tests__/server/audit-log-staging.servertest.tsindex eeae84f685..d75b2e2c30 100644--- a/web/src/__tests__/server/audit-log-staging.servertest.ts+++ b/web/src/__tests__/server/audit-log-staging.servertest.ts@@ -30,6 +30,10 @@ import {} from "@langfuse/shared/src/server";import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";import { auditLog } from "@/src/features/audit-logs/server";+import {+ createDatasetItemForApi,+ deleteDatasetItemForApi,+} from "@/src/features/datasets/server/publicDatasetService";const queryAuditLogs = async (query: string,@@ -295,6 +299,67 @@ describe("audit log staging", () => {return { record, staged };};+ it("omits dataset item payloads from both audit sinks while preserving versions", async () => {+ const { orgId, projectId } = await createOrgProjectAndApiKey();+ const { id: apiKeyId } = await prisma.apiKey.findFirstOrThrow({+ where: { projectId },+ });+ const dataset = await prisma.dataset.create({+ data: { name: randomUUID(), projectId },+ });+ const payload = {+ input: { message: "input" },+ expectedOutput: { answer: "output" },+ metadata: { source: "test" },+ };+ const auditScope = { orgId, projectId, apiKeyId };+ const created = await createDatasetItemForApi({+ projectId,+ auditScope,+ input: { datasetId: dataset.id, ...payload },+ });+ expect(created).toMatchObject(payload);+ const updatedPayload = { ...payload, input: { message: "updated" } };+ const updated = await createDatasetItemForApi({+ projectId,+ auditScope,+ input: { datasetId: dataset.id, id: created.id, ...updatedPayload },+ });+ expect(updated).toMatchObject(updatedPayload);+ await deleteDatasetItemForApi({+ ...auditScope,+ datasetItemId: created.id,+ });++ const versions = await prisma.datasetItem.findMany({+ where: { projectId, id: created.id, isDeleted: false },+ orderBy: { validFrom: "asc" },+ });+ expect(versions).toMatchObject([payload, updatedPayload]);++ const records = await prisma.auditLog.findMany({+ where: { projectId, resourceId: created.id },+ });+ expect(records).toHaveLength(3);+ for (const record of records) {+ expect(record.before ?? record.after).not.toBeNull();+ const id = deriveAuditLogEventId(record.id, record.createdAt);+ stagedIds.push(id);+ const staged = await prisma.auditLogStaging.findUniqueOrThrow({+ where: { id },+ });+ expect(convertAuditLogStagingRowToClickhouse(staged)).toMatchObject({+ before: record.before ?? "",+ after: record.after ?? "",+ });+ for (const snapshot of [record.before, record.after]) {+ if (snapshot === null) continue;+ expect(JSON.parse(snapshot).id).toBe(created.id);+ expect(snapshot).not.toMatch(/"(input|expectedOutput|metadata)":/);+ }+ }+ });+it("stages the change event of the Postgres row under the id derived from it", async () => {const { orgId, projectId } = await createOrgProjectAndApiKey();const resourceId = randomUUID();diff --git a/web/src/__tests__/server/datasets-trpc.servertest.ts b/web/src/__tests__/server/datasets-trpc.servertest.tsindex 40d214fdf0..04a0750f33 100644--- a/web/src/__tests__/server/datasets-trpc.servertest.ts+++ b/web/src/__tests__/server/datasets-trpc.servertest.ts@@ -7,6 +7,10 @@ import type { Session } from "next-auth";import { v4 } from "uuid";import { afterAll, afterEach, describe, expect, it, vi } from "vitest";+vi.mock("@/src/features/audit-logs/isAuditLogEnabled", () => ({+ isAuditLogEnabled: () => true,+}));+const orgIds: string[] = [];async function prepare({@@ -68,6 +72,14 @@ describe("datasets trpc", () => {});afterAll(async () => {+ await prisma.auditLog.deleteMany({ where: { orgId: { in: orgIds } } });+ await prisma.auditLogStaging.deleteMany({+ where: {+ OR: orgIds.map((orgId) => ({+ payload: { path: ["orgId"], equals: orgId },+ })),+ },+ });await prisma.organization.deleteMany({where: {id: { in: orgIds },@@ -75,6 +87,53 @@ describe("datasets trpc", () => {});});+ it("omits dataset item payloads from CRUD and bulk-create audit logs", async () => {+ const { project, caller } = await prepare();+ const dataset = await prisma.dataset.create({+ data: { projectId: project.id, name: v4() },+ });+ const payload = {+ input: { message: "input" },+ expectedOutput: { answer: "output" },+ metadata: { source: "test" },+ };+ const scope = { projectId: project.id, datasetId: dataset.id };+ const input = {+ ...scope,+ input: JSON.stringify(payload.input),+ expectedOutput: JSON.stringify(payload.expectedOutput),+ metadata: JSON.stringify(payload.metadata),+ };+ const created = await caller.datasets.createDatasetItem(input);+ expect(created).toMatchObject(payload);+ await caller.datasets.updateDatasetItem({+ ...input,+ datasetItemId: created.id,+ status: "ARCHIVED",+ });+ await caller.datasets.deleteDatasetItem({+ ...scope,+ datasetItemId: created.id,+ });+ await caller.datasets.createManyDatasetItems({+ projectId: project.id,+ items: [input],+ });++ const records = await prisma.auditLog.findMany({+ where: { projectId: project.id, resourceType: "datasetItem" },+ });+ expect(records).toHaveLength(4);+ for (const record of records) {+ expect(record.before ?? record.after).not.toBeNull();+ for (const snapshot of [record.before, record.after]) {+ if (snapshot === null) continue;+ expect(JSON.parse(snapshot).id).toBe(record.resourceId);+ expect(snapshot).not.toMatch(/"(input|expectedOutput|metadata)":/);+ }+ }+ });+describe("datasets RBAC", () => {it("allows viewers to read dataset metadata", async () => {const { project, caller } = await prepare({diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.tsindex 51f24483d4..4e4b44af7f 100644--- a/web/src/features/datasets/server/dataset-router.ts+++ b/web/src/features/datasets/server/dataset-router.ts@@ -1,5 +1,6 @@/* eslint-disable no-nested-ternary */import { z } from "zod";+import { omit } from "lodash";import {createTRPCRouter,protectedProjectProcedure,@@ -1175,7 +1176,7 @@ export const datasetRouter = createTRPCRouter({resourceType: "datasetItem",resourceId: input.datasetItemId,action: "update",- after: datasetItem,+ after: omit(datasetItem, ["input", "expectedOutput", "metadata"]),});return datasetItem;@@ -1488,7 +1489,11 @@ export const datasetRouter = createTRPCRouter({resourceType: "datasetItem",resourceId: result.deletedItem.id,action: "delete",- before: result.deletedItem,+ before: omit(result.deletedItem, [+ "input",+ "expectedOutput",+ "metadata",+ ]),});return result.deletedItem;@@ -1692,7 +1697,11 @@ export const datasetRouter = createTRPCRouter({resourceType: "datasetItem",resourceId: result.datasetItem.id,action: "create",- after: result.datasetItem,+ after: omit(result.datasetItem, [+ "input",+ "expectedOutput",+ "metadata",+ ]),});return result.datasetItem;@@ -1750,7 +1759,7 @@ export const datasetRouter = createTRPCRouter({resourceType: "datasetItem",resourceId: item.id,action: "create",- after: item,+ after: omit(item, ["input", "expectedOutput", "metadata"]),}),),);diff --git a/web/src/features/datasets/server/publicDatasetService.ts b/web/src/features/datasets/server/publicDatasetService.tsindex 96cc63c26c..5c4adf46ec 100644--- a/web/src/features/datasets/server/publicDatasetService.ts+++ b/web/src/features/datasets/server/publicDatasetService.ts@@ -1,6 +1,7 @@import type { NextApiResponse } from "next";import { createHash } from "node:crypto";import { v4 } from "uuid";+import { omit } from "lodash";import { auditLog } from "@/src/features/audit-logs/server";import { addDatasetRunItemsToEvalQueue } from "@/src/features/evals/server/addDatasetRunItemsToEvalQueue";import {@@ -648,8 +649,10 @@ export const createDatasetItemForApi = async ({projectId,orgId: auditScope.orgId,apiKeyId: auditScope.apiKeyId,- before: existingDatasetItem ?? undefined,- after: datasetItem,+ before: existingDatasetItem+ ? omit(existingDatasetItem, ["input", "expectedOutput", "metadata"])+ : undefined,+ after: omit(datasetItem, ["input", "expectedOutput", "metadata"]),});}@@ -717,7 +720,7 @@ export const deleteDatasetItemForApi = async ({projectId,orgId,apiKeyId,- before: result.deletedItem,+ before: omit(result.deletedItem, ["input", "expectedOutput", "metadata"]),});return {