Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(audit-logs): log actor info for audited actions and deletions (#18107)

Pratima Patelb036980ca137
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.ts
index 531964244f..aab9f82b4e 100644
--- a/packages/shared/src/server/queues.ts
+++ b/packages/shared/src/server/queues.ts
@@ -100,9 +100,19 @@ export const TraceQueueEventSchema = z.object({
exactTimestamp: z.date().optional(),
traceEnvironment: z.string().optional(), // Optional to maintain backward compatibility with existing jobs in queue during deployment. 'optional()' can be removed after queue was exhausted
});
+// Who triggered a queued job (e.g. a deletion). Only used for logging; optional
+// on payloads so that jobs enqueued before this field existed still parse.
+export const ActorSchema = z.object({
+ type: z.enum(["USER", "API_KEY"]),
+ userId: z.string().optional(),
+ apiKeyId: z.string().optional(),
+ publicKey: z.string().optional(),
+});
+export type Actor = z.infer<typeof ActorSchema>;
export const TracesQueueEventSchema = z.object({
projectId: z.string(),
traceIds: z.array(z.string()),
+ actor: ActorSchema.optional(),
});
export const ScoresQueueEventSchema = z.object({
projectId: z.string(),
@@ -185,62 +195,38 @@ export const DataRetentionProcessingEventSchema = z.object({
projectId: z.string(),
retention: z.number(),
});
+// Fields shared by the batch actions that select rows from a table and apply
+// one action to them.
+const BatchActionTableEventBaseSchema = z.object({
+ projectId: z.string(),
+ query: BatchActionQuerySchema,
+ tableName: z.enum(BatchTableNames),
+ cutoffCreatedAt: z.date(),
+ targetId: z.string().optional(),
+ type: z.enum(BatchActionType),
+ userId: z.string().optional(), // Requesting user, for logging only
+});
+
export const BatchActionProcessingEventSchema = z.discriminatedUnion(
"actionId",
[
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("score-delete"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("dataset-delete"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("trace-delete"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("trace-add-to-annotation-queue"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("session-add-to-annotation-queue"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
- z.object({
+ BatchActionTableEventBaseSchema.extend({
actionId: z.literal("observation-add-to-annotation-queue"),
- projectId: z.string(),
- query: BatchActionQuerySchema,
- tableName: z.enum(BatchTableNames),
- cutoffCreatedAt: z.date(),
- targetId: z.string().optional(),
- type: z.enum(BatchActionType),
}),
z.object({
actionId: z.literal("eval-create"),
diff --git a/packages/shared/src/server/traceDeletionProcessor.ts b/packages/shared/src/server/traceDeletionProcessor.ts
index 6f21be57cd..c7e3069937 100644
--- a/packages/shared/src/server/traceDeletionProcessor.ts
+++ b/packages/shared/src/server/traceDeletionProcessor.ts
@@ -1,13 +1,15 @@
import { randomUUID } from "crypto";
import { prisma } from "../db";
import { TraceDeleteQueue } from "./redis/traceDelete";
-import { QueueJobs } from "./queues";
+import { type Actor, QueueJobs } from "./queues";
import { logger } from "./logger";
import { env } from "../env";
import { shouldSkipDeletionFor } from "./deletionGuard";
+import { formatSubmittedPublicKeyForLog } from "./auth/apiKeys";
export interface TraceDeletionProcessorOptions {
delayMs?: number; // Default from LANGFUSE_TRACE_DELETE_DELAY_MS env var
+ actor?: Actor; // Who requested the deletion, logged in web and worker
}
/**
@@ -28,7 +30,7 @@ export async function traceDeletionProcessor(
traceIds: string[],
options: TraceDeletionProcessorOptions = {},
): Promise<void> {
- const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS } = options;
+ const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS, actor } = options;
if (traceIds.length === 0) {
logger.warn("traceDeletionProcessor called with empty traceIds array", {
@@ -38,11 +40,14 @@ export async function traceDeletionProcessor(
}
logger.info(
- `Processing trace deletion for ${traceIds.length} traces in project ${projectId}`,
+ `Processing trace deletion for ${traceIds.length} traces in project ${projectId}${
+ actor ? ` requested by ${formatActor(actor)}` : ""
+ }`,
{
projectId,
traceIds,
delayMs,
+ actor: getActorLogMetadata(actor),
},
);
@@ -78,6 +83,7 @@ export async function traceDeletionProcessor(
payload: {
projectId,
traceIds,
+ actor,
},
},
{
@@ -93,3 +99,20 @@ export async function traceDeletionProcessor(
throw error;
}
}
+
+export function formatActor(actor: Actor): string {
+ if (actor.type === "API_KEY") {
+ if (actor.publicKey) {
+ return `API key ${formatSubmittedPublicKeyForLog(actor.publicKey)}`;
+ }
+ return `API key ${actor.apiKeyId ?? "unknown"}`;
+ }
+ return `user ${actor.userId ?? "unknown"}`;
+}
+
+// The submitted public key is client-controlled, so log metadata only carries
+// the database ids; formatActor prints the sanitized public key.
+export function getActorLogMetadata(actor: Actor | undefined) {
+ if (!actor) return undefined;
+ return { type: actor.type, userId: actor.userId, apiKeyId: actor.apiKeyId };
+}
diff --git a/web/src/features/audit-logs/auditLog.ts b/web/src/features/audit-logs/auditLog.ts
index 18437b3c17..4513e26ce5 100644
--- a/web/src/features/audit-logs/auditLog.ts
+++ b/web/src/features/audit-logs/auditLog.ts
@@ -4,6 +4,10 @@ import {
AuditLogRecordType,
type Prisma,
} from "@langfuse/shared/src/db";
+import {
+ formatSubmittedPublicKeyForLog,
+ logger,
+} from "@langfuse/shared/src/server";
type AuditableResource =
| "annotationQueue"
@@ -87,6 +91,44 @@ type AuditLog = {
}
);
+// Mirrors each audit log record into the application logs so that actors can be
+// correlated with web/worker log lines (e.g. trace deletions) without querying
+// the audit_logs table. Only ids are logged, no emails or names.
+function logAuditEvent(
+ log: AuditLog,
+ actor: {
+ type: AuditLogRecordType;
+ orgId: string;
+ projectId?: string;
+ userId?: string;
+ apiKeyId?: string;
+ publicKey?: string;
+ },
+) {
+ let actorLabel = actor.userId;
+ if (actor.type === AuditLogRecordType.API_KEY) {
+ actorLabel = actor.publicKey
+ ? formatSubmittedPublicKeyForLog(actor.publicKey)
+ : actor.apiKeyId;
+ }
+
+ logger.info(
+ `Audit log: ${log.resourceType}.${log.action} ${log.resourceId} by ${actor.type} ${actorLabel}`,
+ {
+ auditLog: true,
+ resourceType: log.resourceType,
+ resourceId: log.resourceId,
+ action: log.action,
+ actorType: actor.type,
+ userId: actor.userId,
+ apiKeyId: actor.apiKeyId,
+ publicKey: actor.publicKey,
+ orgId: actor.orgId,
+ projectId: actor.projectId,
+ },
+ );
+}
+
export async function auditLog(
log: AuditLog,
prisma?: typeof _prisma | Prisma.TransactionClient,
@@ -109,16 +151,19 @@ export async function auditLog(
select: {
isInAppAgentKey: true,
createdByUserId: true,
+ publicKey: true,
},
});
+ const userId =
+ apiKey?.isInAppAgentKey === true
+ ? (apiKey.createdByUserId ?? undefined)
+ : undefined;
+
await db.auditLog.create({
data: {
apiKeyId: log.apiKeyId,
- userId:
- apiKey?.isInAppAgentKey === true
- ? (apiKey.createdByUserId ?? undefined)
- : undefined,
+ userId,
orgId: log.orgId,
projectId: log.projectId,
type: AuditLogRecordType.API_KEY,
@@ -126,6 +171,15 @@ export async function auditLog(
},
});
+ logAuditEvent(log, {
+ type: AuditLogRecordType.API_KEY,
+ orgId: log.orgId,
+ projectId: log.projectId,
+ userId,
+ apiKeyId: log.apiKeyId,
+ publicKey: apiKey?.publicKey,
+ });
+
return;
}
@@ -142,6 +196,13 @@ export async function auditLog(
},
});
+ logAuditEvent(log, {
+ type: AuditLogRecordType.USER,
+ orgId: log.session.orgId,
+ projectId: log.session.projectId,
+ userId: log.session.user.id,
+ });
+
return;
}
@@ -158,6 +219,13 @@ export async function auditLog(
},
});
+ logAuditEvent(log, {
+ type: AuditLogRecordType.USER,
+ orgId: log.orgId,
+ projectId: log.projectId,
+ userId: log.userId,
+ });
+
return;
}
}
diff --git a/web/src/features/table/server/createBatchActionJob.ts b/web/src/features/table/server/createBatchActionJob.ts
index 130a6eb0cd..d2f30f707b 100644
--- a/web/src/features/table/server/createBatchActionJob.ts
+++ b/web/src/features/table/server/createBatchActionJob.ts
@@ -46,6 +46,24 @@ type CreateBatchActionJob = {
useEventsTableOverride?: boolean;
};
+const logBatchActionCreated = (params: {
+ batchActionId: string;
+ projectId: string;
+ actionId: string;
+ userId: string;
+}) => {
+ logger.info(
+ `Batch action ${params.actionId} created in project ${params.projectId} by user ${params.userId}`,
+ {
+ batchActionId: params.batchActionId,
+ projectId: params.projectId,
+ actionId: params.actionId,
+ actorType: "USER",
+ userId: params.userId,
+ },
+ );
+};
+
const ACTIVE_BATCH_ACTION_STATUSES = [
BatchActionStatus.Queued,
BatchActionStatus.Processing,
@@ -152,6 +170,13 @@ export const createBatchActionJob = async ({
action: actionType as string,
});
+ logBatchActionCreated({
+ batchActionId,
+ projectId,
+ actionId,
+ userId: session.user.id,
+ });
+
return;
}
@@ -188,6 +213,7 @@ export const createBatchActionJob = async ({
query: queryWithSnapshot,
targetId: targetId,
type: actionType,
+ userId: session.user.id,
},
},
{
@@ -195,5 +221,12 @@ export const createBatchActionJob = async ({
},
);
+ logBatchActionCreated({
+ batchActionId,
+ projectId,
+ actionId,
+ userId: session.user.id,
+ });
+
return;
};
diff --git a/web/src/pages/api/public/traces/[traceId].ts b/web/src/pages/api/public/traces/[traceId].ts
index 3babd8e193..a789de5e42 100644
--- a/web/src/pages/api/public/traces/[traceId].ts
+++ b/web/src/pages/api/public/traces/[traceId].ts
@@ -224,7 +224,13 @@ export default withMiddlewares(
orgId: auth.scope.orgId,
});
- await traceDeletionProcessor(auth.scope.projectId, [traceId]);
+ await traceDeletionProcessor(auth.scope.projectId, [traceId], {
+ actor: {
+ type: "API_KEY",
+ apiKeyId: auth.scope.apiKeyId,
+ publicKey: auth.scope.publicKey,
+ },
+ });
return { message: "Trace deleted successfully" };
},
diff --git a/web/src/pages/api/public/traces/index.ts b/web/src/pages/api/public/traces/index.ts
index ebeba8a4e3..35eb902788 100644
--- a/web/src/pages/api/public/traces/index.ts
+++ b/web/src/pages/api/public/traces/index.ts
@@ -247,7 +247,13 @@ export default withMiddlewares(
),
);
- await traceDeletionProcessor(auth.scope.projectId, traceIds);
+ await traceDeletionProcessor(auth.scope.projectId, traceIds, {
+ actor: {
+ type: "API_KEY",
+ apiKeyId: auth.scope.apiKeyId,
+ publicKey: auth.scope.publicKey,
+ },
+ });
return { message: "Traces deleted successfully" };
},
diff --git a/web/src/server/api/routers/traces.ts b/web/src/server/api/routers/traces.ts
index e2ad498acf..135eaa01c6 100644
--- a/web/src/server/api/routers/traces.ts
+++ b/web/src/server/api/routers/traces.ts
@@ -560,7 +560,9 @@ export const traceRouter = createTRPCRouter({
),
);
- await traceDeletionProcessor(input.projectId, input.traceIds);
+ await traceDeletionProcessor(input.projectId, input.traceIds, {
+ actor: { type: "USER", userId: ctx.session.user.id },
+ });
}
}),
bookmark: protectedProjectProcedure
diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.ts
index 0898d11b06..40fc9fc6a3 100644
--- a/worker/src/features/batchAction/handleBatchActionJob.ts
+++ b/worker/src/features/batchAction/handleBatchActionJob.ts
@@ -99,13 +99,17 @@ async function processActionChunk(
chunkIds: string[],
projectId: string,
targetId?: string,
+ userId?: string,
): Promise<void> {
try {
switch (actionId) {
case "trace-delete":
// Legacy queue path. Durable trace-delete BatchActions are processed
// by TraceDeleteBatchActionRunner.
- await traceDeletionProcessor(projectId, chunkIds, { delayMs: 0 });
+ await traceDeletionProcessor(projectId, chunkIds, {
+ delayMs: 0,
+ actor: userId ? { type: "USER", userId } : undefined,
+ });
break;
case "trace-add-to-annotation-queue":
@@ -129,10 +133,18 @@ async function processActionChunk(
break;
case "score-delete":
+ logger.info(
+ `Batch deleting ${chunkIds.length} scores in project ${projectId} requested by user ${userId ?? "unknown"}`,
+ { projectId, scoreIds: chunkIds, actorType: "USER", userId },
+ );
await processClickhouseScoreDelete(projectId, chunkIds);
break;
case "dataset-delete":
+ logger.info(
+ `Batch deleting ${chunkIds.length} datasets in project ${projectId} requested by user ${userId ?? "unknown"}`,
+ { projectId, datasetIds: chunkIds, actorType: "USER", userId },
+ );
await processDeleteDatasets(projectId, chunkIds);
break;
@@ -213,8 +225,20 @@ export const handleBatchActionJob = async (
actionId === "score-delete" ||
actionId === "dataset-delete"
) {
- const { projectId, tableName, query, cutoffCreatedAt, targetId, type } =
- batchActionEvent;
+ const {
+ projectId,
+ tableName,
+ query,
+ cutoffCreatedAt,
+ targetId,
+ type,
+ userId,
+ } = batchActionEvent;
+
+ logger.info(
+ `Processing batch action ${actionId} in project ${projectId} requested by user ${userId ?? "unknown"}`,
+ { projectId, actionId, actorType: "USER", userId },
+ );
if (type === BatchActionType.Create && !targetId) {
throw new Error(`Target ID is required for create action`);
@@ -280,6 +304,7 @@ export const handleBatchActionJob = async (
batch.map((r) => r.id),
projectId,
targetId,
+ userId,
);
}
} else if (actionId === "eval-create") {
diff --git a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
index 2a0c94f7fa..ae1b59b034 100644
--- a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
+++ b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
@@ -389,12 +389,18 @@ export const processTraceDeleteBatchAction = async ({
});
}
- logger.info("Processing trace delete batch action page", {
- batchActionId,
- projectId: batchAction.projectId,
- source: state.config.source,
- traceCount: inFlightBatch.traceIds.length,
- });
+ logger.info(
+ `Processing trace delete batch action page requested by user ${batchAction.userId}`,
+ {
+ batchActionId,
+ projectId: batchAction.projectId,
+ source: state.config.source,
+ traceCount: inFlightBatch.traceIds.length,
+ traceIds: inFlightBatch.traceIds,
+ actorType: "USER",
+ userId: batchAction.userId,
+ },
+ );
if (
await shouldSkipDeletion(
diff --git a/worker/src/features/trace-delete-batch-action-runner/index.ts b/worker/src/features/trace-delete-batch-action-runner/index.ts
index af00ab945c..bd7ab4b3b5 100644
--- a/worker/src/features/trace-delete-batch-action-runner/index.ts
+++ b/worker/src/features/trace-delete-batch-action-runner/index.ts
@@ -33,6 +33,7 @@ const ACTIVE_TRACE_DELETE_BATCH_ACTION_WHERE = {
type ActiveTraceDeleteBatchAction = {
id: string;
projectId: string;
+ userId: string;
status: string;
processedCount: number | null;
failedCount: number | null;
@@ -137,6 +138,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {
logger.info(`${this.instanceName}: Processing trace delete action`, {
batchActionId: batchAction.id,
projectId: batchAction.projectId,
+ userId: batchAction.userId,
status: batchAction.status,
source,
activeActionCount,
@@ -195,6 +197,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {
select: {
id: true,
projectId: true,
+ userId: true,
status: true,
processedCount: true,
failedCount: true,
diff --git a/worker/src/queues/traceDelete.ts b/worker/src/queues/traceDelete.ts
index 8cc456f654..29b89e05f3 100644
--- a/worker/src/queues/traceDelete.ts
+++ b/worker/src/queues/traceDelete.ts
@@ -1,5 +1,7 @@
import { Job, Processor } from "bullmq";
import {
+ formatActor,
+ getActorLogMetadata,
getCurrentSpan,
logger,
QueueName,
@@ -21,6 +23,22 @@ export const traceDeleteProcessor: Processor = async (
? job.data.payload.traceIds
: [job.data.payload.traceId];
+ const actor =
+ "actor" in job.data.payload ? job.data.payload.actor : undefined;
+
+ if (actor) {
+ // The job may also pick up pending deletions requested by other actors;
+ // the actor applies to the trace ids from this event only.
+ logger.info(
+ `Trace deletion job for ${eventTraceIds.length} traces in project ${projectId} requested by ${formatActor(actor)}`,
+ {
+ projectId,
+ traceIds: eventTraceIds,
+ actor: getActorLogMetadata(actor),
+ },
+ );
+ }
+
const span = getCurrentSpan();
const [toBeDeletedTraces, pendingEventTraceIds] = await Promise.all([