Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(audit-logs): omit dataset item payloads from audit logs (#19111)

marliessophiea8da34e054d2
diff --git a/web/src/__tests__/server/audit-log-staging.servertest.ts b/web/src/__tests__/server/audit-log-staging.servertest.ts
index eeae84f685..d75b2e2c30 100644
--- a/web/src/__tests__/server/audit-log-staging.servertest.ts
+++ b/web/src/__tests__/server/audit-log-staging.servertest.ts
@@ -30,6 +30,10 @@ import {
} from "@langfuse/shared/src/server";
import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";
import { auditLog } from "@/src/features/audit-logs/server";
+import {
+ createDatasetItemForApi,
+ deleteDatasetItemForApi,
+} from "@/src/features/datasets/server/publicDatasetService";
const queryAuditLogs = async (
query: string,
@@ -295,6 +299,67 @@ describe("audit log staging", () => {
return { record, staged };
};
+ it("omits dataset item payloads from both audit sinks while preserving versions", async () => {
+ const { orgId, projectId } = await createOrgProjectAndApiKey();
+ const { id: apiKeyId } = await prisma.apiKey.findFirstOrThrow({
+ where: { projectId },
+ });
+ const dataset = await prisma.dataset.create({
+ data: { name: randomUUID(), projectId },
+ });
+ const payload = {
+ input: { message: "input" },
+ expectedOutput: { answer: "output" },
+ metadata: { source: "test" },
+ };
+ const auditScope = { orgId, projectId, apiKeyId };
+ const created = await createDatasetItemForApi({
+ projectId,
+ auditScope,
+ input: { datasetId: dataset.id, ...payload },
+ });
+ expect(created).toMatchObject(payload);
+ const updatedPayload = { ...payload, input: { message: "updated" } };
+ const updated = await createDatasetItemForApi({
+ projectId,
+ auditScope,
+ input: { datasetId: dataset.id, id: created.id, ...updatedPayload },
+ });
+ expect(updated).toMatchObject(updatedPayload);
+ await deleteDatasetItemForApi({
+ ...auditScope,
+ datasetItemId: created.id,
+ });
+
+ const versions = await prisma.datasetItem.findMany({
+ where: { projectId, id: created.id, isDeleted: false },
+ orderBy: { validFrom: "asc" },
+ });
+ expect(versions).toMatchObject([payload, updatedPayload]);
+
+ const records = await prisma.auditLog.findMany({
+ where: { projectId, resourceId: created.id },
+ });
+ expect(records).toHaveLength(3);
+ for (const record of records) {
+ expect(record.before ?? record.after).not.toBeNull();
+ const id = deriveAuditLogEventId(record.id, record.createdAt);
+ stagedIds.push(id);
+ const staged = await prisma.auditLogStaging.findUniqueOrThrow({
+ where: { id },
+ });
+ expect(convertAuditLogStagingRowToClickhouse(staged)).toMatchObject({
+ before: record.before ?? "",
+ after: record.after ?? "",
+ });
+ for (const snapshot of [record.before, record.after]) {
+ if (snapshot === null) continue;
+ expect(JSON.parse(snapshot).id).toBe(created.id);
+ expect(snapshot).not.toMatch(/"(input|expectedOutput|metadata)":/);
+ }
+ }
+ });
+
it("stages the change event of the Postgres row under the id derived from it", async () => {
const { orgId, projectId } = await createOrgProjectAndApiKey();
const resourceId = randomUUID();
diff --git a/web/src/__tests__/server/datasets-trpc.servertest.ts b/web/src/__tests__/server/datasets-trpc.servertest.ts
index 40d214fdf0..04a0750f33 100644
--- a/web/src/__tests__/server/datasets-trpc.servertest.ts
+++ b/web/src/__tests__/server/datasets-trpc.servertest.ts
@@ -7,6 +7,10 @@ import type { Session } from "next-auth";
import { v4 } from "uuid";
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
+vi.mock("@/src/features/audit-logs/isAuditLogEnabled", () => ({
+ isAuditLogEnabled: () => true,
+}));
+
const orgIds: string[] = [];
async function prepare({
@@ -68,6 +72,14 @@ describe("datasets trpc", () => {
});
afterAll(async () => {
+ await prisma.auditLog.deleteMany({ where: { orgId: { in: orgIds } } });
+ await prisma.auditLogStaging.deleteMany({
+ where: {
+ OR: orgIds.map((orgId) => ({
+ payload: { path: ["orgId"], equals: orgId },
+ })),
+ },
+ });
await prisma.organization.deleteMany({
where: {
id: { in: orgIds },
@@ -75,6 +87,53 @@ describe("datasets trpc", () => {
});
});
+ it("omits dataset item payloads from CRUD and bulk-create audit logs", async () => {
+ const { project, caller } = await prepare();
+ const dataset = await prisma.dataset.create({
+ data: { projectId: project.id, name: v4() },
+ });
+ const payload = {
+ input: { message: "input" },
+ expectedOutput: { answer: "output" },
+ metadata: { source: "test" },
+ };
+ const scope = { projectId: project.id, datasetId: dataset.id };
+ const input = {
+ ...scope,
+ input: JSON.stringify(payload.input),
+ expectedOutput: JSON.stringify(payload.expectedOutput),
+ metadata: JSON.stringify(payload.metadata),
+ };
+ const created = await caller.datasets.createDatasetItem(input);
+ expect(created).toMatchObject(payload);
+ await caller.datasets.updateDatasetItem({
+ ...input,
+ datasetItemId: created.id,
+ status: "ARCHIVED",
+ });
+ await caller.datasets.deleteDatasetItem({
+ ...scope,
+ datasetItemId: created.id,
+ });
+ await caller.datasets.createManyDatasetItems({
+ projectId: project.id,
+ items: [input],
+ });
+
+ const records = await prisma.auditLog.findMany({
+ where: { projectId: project.id, resourceType: "datasetItem" },
+ });
+ expect(records).toHaveLength(4);
+ for (const record of records) {
+ expect(record.before ?? record.after).not.toBeNull();
+ for (const snapshot of [record.before, record.after]) {
+ if (snapshot === null) continue;
+ expect(JSON.parse(snapshot).id).toBe(record.resourceId);
+ expect(snapshot).not.toMatch(/"(input|expectedOutput|metadata)":/);
+ }
+ }
+ });
+
describe("datasets RBAC", () => {
it("allows viewers to read dataset metadata", async () => {
const { project, caller } = await prepare({
diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.ts
index 51f24483d4..4e4b44af7f 100644
--- a/web/src/features/datasets/server/dataset-router.ts
+++ b/web/src/features/datasets/server/dataset-router.ts
@@ -1,5 +1,6 @@
/* eslint-disable no-nested-ternary */
import { z } from "zod";
+import { omit } from "lodash";
import {
createTRPCRouter,
protectedProjectProcedure,
@@ -1175,7 +1176,7 @@ export const datasetRouter = createTRPCRouter({
resourceType: "datasetItem",
resourceId: input.datasetItemId,
action: "update",
- after: datasetItem,
+ after: omit(datasetItem, ["input", "expectedOutput", "metadata"]),
});
return datasetItem;
@@ -1488,7 +1489,11 @@ export const datasetRouter = createTRPCRouter({
resourceType: "datasetItem",
resourceId: result.deletedItem.id,
action: "delete",
- before: result.deletedItem,
+ before: omit(result.deletedItem, [
+ "input",
+ "expectedOutput",
+ "metadata",
+ ]),
});
return result.deletedItem;
@@ -1692,7 +1697,11 @@ export const datasetRouter = createTRPCRouter({
resourceType: "datasetItem",
resourceId: result.datasetItem.id,
action: "create",
- after: result.datasetItem,
+ after: omit(result.datasetItem, [
+ "input",
+ "expectedOutput",
+ "metadata",
+ ]),
});
return result.datasetItem;
@@ -1750,7 +1759,7 @@ export const datasetRouter = createTRPCRouter({
resourceType: "datasetItem",
resourceId: item.id,
action: "create",
- after: item,
+ after: omit(item, ["input", "expectedOutput", "metadata"]),
}),
),
);
diff --git a/web/src/features/datasets/server/publicDatasetService.ts b/web/src/features/datasets/server/publicDatasetService.ts
index 96cc63c26c..5c4adf46ec 100644
--- a/web/src/features/datasets/server/publicDatasetService.ts
+++ b/web/src/features/datasets/server/publicDatasetService.ts
@@ -1,6 +1,7 @@
import type { NextApiResponse } from "next";
import { createHash } from "node:crypto";
import { v4 } from "uuid";
+import { omit } from "lodash";
import { auditLog } from "@/src/features/audit-logs/server";
import { addDatasetRunItemsToEvalQueue } from "@/src/features/evals/server/addDatasetRunItemsToEvalQueue";
import {
@@ -648,8 +649,10 @@ export const createDatasetItemForApi = async ({
projectId,
orgId: auditScope.orgId,
apiKeyId: auditScope.apiKeyId,
- before: existingDatasetItem ?? undefined,
- after: datasetItem,
+ before: existingDatasetItem
+ ? omit(existingDatasetItem, ["input", "expectedOutput", "metadata"])
+ : undefined,
+ after: omit(datasetItem, ["input", "expectedOutput", "metadata"]),
});
}
@@ -717,7 +720,7 @@ export const deleteDatasetItemForApi = async ({
projectId,
orgId,
apiKeyId,
- before: result.deletedItem,
+ before: omit(result.deletedItem, ["input", "expectedOutput", "metadata"]),
});
return {