🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(audit-logs): log actor info for audited actions and deletions (#18107)
Pratima Patelb036980ca137
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 531964244f..aab9f82b4e 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -100,9 +100,19 @@ export const TraceQueueEventSchema = z.object({exactTimestamp: z.date().optional(),traceEnvironment: z.string().optional(), // Optional to maintain backward compatibility with existing jobs in queue during deployment. 'optional()' can be removed after queue was exhausted});+// Who triggered a queued job (e.g. a deletion). Only used for logging; optional+// on payloads so that jobs enqueued before this field existed still parse.+export const ActorSchema = z.object({+ type: z.enum(["USER", "API_KEY"]),+ userId: z.string().optional(),+ apiKeyId: z.string().optional(),+ publicKey: z.string().optional(),+});+export type Actor = z.infer<typeof ActorSchema>;export const TracesQueueEventSchema = z.object({projectId: z.string(),traceIds: z.array(z.string()),+ actor: ActorSchema.optional(),});export const ScoresQueueEventSchema = z.object({projectId: z.string(),@@ -185,62 +195,38 @@ export const DataRetentionProcessingEventSchema = z.object({projectId: z.string(),retention: z.number(),});+// Fields shared by the batch actions that select rows from a table and apply+// one action to them.+const BatchActionTableEventBaseSchema = z.object({+ projectId: z.string(),+ query: BatchActionQuerySchema,+ tableName: z.enum(BatchTableNames),+ cutoffCreatedAt: z.date(),+ targetId: z.string().optional(),+ type: z.enum(BatchActionType),+ userId: z.string().optional(), // Requesting user, for logging only+});+export const BatchActionProcessingEventSchema = z.discriminatedUnion("actionId",[- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("score-delete"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("dataset-delete"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("trace-delete"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("trace-add-to-annotation-queue"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("session-add-to-annotation-queue"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),- z.object({+ BatchActionTableEventBaseSchema.extend({actionId: z.literal("observation-add-to-annotation-queue"),- projectId: z.string(),- query: BatchActionQuerySchema,- tableName: z.enum(BatchTableNames),- cutoffCreatedAt: z.date(),- targetId: z.string().optional(),- type: z.enum(BatchActionType),}),z.object({actionId: z.literal("eval-create"),diff --git a/packages/shared/src/server/traceDeletionProcessor.ts b/packages/shared/src/server/traceDeletionProcessor.tsindex 6f21be57cd..c7e3069937 100644--- a/packages/shared/src/server/traceDeletionProcessor.ts+++ b/packages/shared/src/server/traceDeletionProcessor.ts@@ -1,13 +1,15 @@import { randomUUID } from "crypto";import { prisma } from "../db";import { TraceDeleteQueue } from "./redis/traceDelete";-import { QueueJobs } from "./queues";+import { type Actor, QueueJobs } from "./queues";import { logger } from "./logger";import { env } from "../env";import { shouldSkipDeletionFor } from "./deletionGuard";+import { formatSubmittedPublicKeyForLog } from "./auth/apiKeys";export interface TraceDeletionProcessorOptions {delayMs?: number; // Default from LANGFUSE_TRACE_DELETE_DELAY_MS env var+ actor?: Actor; // Who requested the deletion, logged in web and worker}/**@@ -28,7 +30,7 @@ export async function traceDeletionProcessor(traceIds: string[],options: TraceDeletionProcessorOptions = {},): Promise<void> {- const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS } = options;+ const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS, actor } = options;if (traceIds.length === 0) {logger.warn("traceDeletionProcessor called with empty traceIds array", {@@ -38,11 +40,14 @@ export async function traceDeletionProcessor(}logger.info(- `Processing trace deletion for ${traceIds.length} traces in project ${projectId}`,+ `Processing trace deletion for ${traceIds.length} traces in project ${projectId}${+ actor ? ` requested by ${formatActor(actor)}` : ""+ }`,{projectId,traceIds,delayMs,+ actor: getActorLogMetadata(actor),},);@@ -78,6 +83,7 @@ export async function traceDeletionProcessor(payload: {projectId,traceIds,+ actor,},},{@@ -93,3 +99,20 @@ export async function traceDeletionProcessor(throw error;}}++export function formatActor(actor: Actor): string {+ if (actor.type === "API_KEY") {+ if (actor.publicKey) {+ return `API key ${formatSubmittedPublicKeyForLog(actor.publicKey)}`;+ }+ return `API key ${actor.apiKeyId ?? "unknown"}`;+ }+ return `user ${actor.userId ?? "unknown"}`;+}++// The submitted public key is client-controlled, so log metadata only carries+// the database ids; formatActor prints the sanitized public key.+export function getActorLogMetadata(actor: Actor | undefined) {+ if (!actor) return undefined;+ return { type: actor.type, userId: actor.userId, apiKeyId: actor.apiKeyId };+}diff --git a/web/src/features/audit-logs/auditLog.ts b/web/src/features/audit-logs/auditLog.tsindex 18437b3c17..4513e26ce5 100644--- a/web/src/features/audit-logs/auditLog.ts+++ b/web/src/features/audit-logs/auditLog.ts@@ -4,6 +4,10 @@ import {AuditLogRecordType,type Prisma,} from "@langfuse/shared/src/db";+import {+ formatSubmittedPublicKeyForLog,+ logger,+} from "@langfuse/shared/src/server";type AuditableResource =| "annotationQueue"@@ -87,6 +91,44 @@ type AuditLog = {});+// Mirrors each audit log record into the application logs so that actors can be+// correlated with web/worker log lines (e.g. trace deletions) without querying+// the audit_logs table. Only ids are logged, no emails or names.+function logAuditEvent(+ log: AuditLog,+ actor: {+ type: AuditLogRecordType;+ orgId: string;+ projectId?: string;+ userId?: string;+ apiKeyId?: string;+ publicKey?: string;+ },+) {+ let actorLabel = actor.userId;+ if (actor.type === AuditLogRecordType.API_KEY) {+ actorLabel = actor.publicKey+ ? formatSubmittedPublicKeyForLog(actor.publicKey)+ : actor.apiKeyId;+ }++ logger.info(+ `Audit log: ${log.resourceType}.${log.action} ${log.resourceId} by ${actor.type} ${actorLabel}`,+ {+ auditLog: true,+ resourceType: log.resourceType,+ resourceId: log.resourceId,+ action: log.action,+ actorType: actor.type,+ userId: actor.userId,+ apiKeyId: actor.apiKeyId,+ publicKey: actor.publicKey,+ orgId: actor.orgId,+ projectId: actor.projectId,+ },+ );+}+export async function auditLog(log: AuditLog,prisma?: typeof _prisma | Prisma.TransactionClient,@@ -109,16 +151,19 @@ export async function auditLog(select: {isInAppAgentKey: true,createdByUserId: true,+ publicKey: true,},});+ const userId =+ apiKey?.isInAppAgentKey === true+ ? (apiKey.createdByUserId ?? undefined)+ : undefined;+await db.auditLog.create({data: {apiKeyId: log.apiKeyId,- userId:- apiKey?.isInAppAgentKey === true- ? (apiKey.createdByUserId ?? undefined)- : undefined,+ userId,orgId: log.orgId,projectId: log.projectId,type: AuditLogRecordType.API_KEY,@@ -126,6 +171,15 @@ export async function auditLog(},});+ logAuditEvent(log, {+ type: AuditLogRecordType.API_KEY,+ orgId: log.orgId,+ projectId: log.projectId,+ userId,+ apiKeyId: log.apiKeyId,+ publicKey: apiKey?.publicKey,+ });+return;}@@ -142,6 +196,13 @@ export async function auditLog(},});+ logAuditEvent(log, {+ type: AuditLogRecordType.USER,+ orgId: log.session.orgId,+ projectId: log.session.projectId,+ userId: log.session.user.id,+ });+return;}@@ -158,6 +219,13 @@ export async function auditLog(},});+ logAuditEvent(log, {+ type: AuditLogRecordType.USER,+ orgId: log.orgId,+ projectId: log.projectId,+ userId: log.userId,+ });+return;}}diff --git a/web/src/features/table/server/createBatchActionJob.ts b/web/src/features/table/server/createBatchActionJob.tsindex 130a6eb0cd..d2f30f707b 100644--- a/web/src/features/table/server/createBatchActionJob.ts+++ b/web/src/features/table/server/createBatchActionJob.ts@@ -46,6 +46,24 @@ type CreateBatchActionJob = {useEventsTableOverride?: boolean;};+const logBatchActionCreated = (params: {+ batchActionId: string;+ projectId: string;+ actionId: string;+ userId: string;+}) => {+ logger.info(+ `Batch action ${params.actionId} created in project ${params.projectId} by user ${params.userId}`,+ {+ batchActionId: params.batchActionId,+ projectId: params.projectId,+ actionId: params.actionId,+ actorType: "USER",+ userId: params.userId,+ },+ );+};+const ACTIVE_BATCH_ACTION_STATUSES = [BatchActionStatus.Queued,BatchActionStatus.Processing,@@ -152,6 +170,13 @@ export const createBatchActionJob = async ({action: actionType as string,});+ logBatchActionCreated({+ batchActionId,+ projectId,+ actionId,+ userId: session.user.id,+ });+return;}@@ -188,6 +213,7 @@ export const createBatchActionJob = async ({query: queryWithSnapshot,targetId: targetId,type: actionType,+ userId: session.user.id,},},{@@ -195,5 +221,12 @@ export const createBatchActionJob = async ({},);+ logBatchActionCreated({+ batchActionId,+ projectId,+ actionId,+ userId: session.user.id,+ });+return;};diff --git a/web/src/pages/api/public/traces/[traceId].ts b/web/src/pages/api/public/traces/[traceId].tsindex 3babd8e193..a789de5e42 100644--- a/web/src/pages/api/public/traces/[traceId].ts+++ b/web/src/pages/api/public/traces/[traceId].ts@@ -224,7 +224,13 @@ export default withMiddlewares(orgId: auth.scope.orgId,});- await traceDeletionProcessor(auth.scope.projectId, [traceId]);+ await traceDeletionProcessor(auth.scope.projectId, [traceId], {+ actor: {+ type: "API_KEY",+ apiKeyId: auth.scope.apiKeyId,+ publicKey: auth.scope.publicKey,+ },+ });return { message: "Trace deleted successfully" };},diff --git a/web/src/pages/api/public/traces/index.ts b/web/src/pages/api/public/traces/index.tsindex ebeba8a4e3..35eb902788 100644--- a/web/src/pages/api/public/traces/index.ts+++ b/web/src/pages/api/public/traces/index.ts@@ -247,7 +247,13 @@ export default withMiddlewares(),);- await traceDeletionProcessor(auth.scope.projectId, traceIds);+ await traceDeletionProcessor(auth.scope.projectId, traceIds, {+ actor: {+ type: "API_KEY",+ apiKeyId: auth.scope.apiKeyId,+ publicKey: auth.scope.publicKey,+ },+ });return { message: "Traces deleted successfully" };},diff --git a/web/src/server/api/routers/traces.ts b/web/src/server/api/routers/traces.tsindex e2ad498acf..135eaa01c6 100644--- a/web/src/server/api/routers/traces.ts+++ b/web/src/server/api/routers/traces.ts@@ -560,7 +560,9 @@ export const traceRouter = createTRPCRouter({),);- await traceDeletionProcessor(input.projectId, input.traceIds);+ await traceDeletionProcessor(input.projectId, input.traceIds, {+ actor: { type: "USER", userId: ctx.session.user.id },+ });}}),bookmark: protectedProjectProcedurediff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.tsindex 0898d11b06..40fc9fc6a3 100644--- a/worker/src/features/batchAction/handleBatchActionJob.ts+++ b/worker/src/features/batchAction/handleBatchActionJob.ts@@ -99,13 +99,17 @@ async function processActionChunk(chunkIds: string[],projectId: string,targetId?: string,+ userId?: string,): Promise<void> {try {switch (actionId) {case "trace-delete":// Legacy queue path. Durable trace-delete BatchActions are processed// by TraceDeleteBatchActionRunner.- await traceDeletionProcessor(projectId, chunkIds, { delayMs: 0 });+ await traceDeletionProcessor(projectId, chunkIds, {+ delayMs: 0,+ actor: userId ? { type: "USER", userId } : undefined,+ });break;case "trace-add-to-annotation-queue":@@ -129,10 +133,18 @@ async function processActionChunk(break;case "score-delete":+ logger.info(+ `Batch deleting ${chunkIds.length} scores in project ${projectId} requested by user ${userId ?? "unknown"}`,+ { projectId, scoreIds: chunkIds, actorType: "USER", userId },+ );await processClickhouseScoreDelete(projectId, chunkIds);break;case "dataset-delete":+ logger.info(+ `Batch deleting ${chunkIds.length} datasets in project ${projectId} requested by user ${userId ?? "unknown"}`,+ { projectId, datasetIds: chunkIds, actorType: "USER", userId },+ );await processDeleteDatasets(projectId, chunkIds);break;@@ -213,8 +225,20 @@ export const handleBatchActionJob = async (actionId === "score-delete" ||actionId === "dataset-delete") {- const { projectId, tableName, query, cutoffCreatedAt, targetId, type } =- batchActionEvent;+ const {+ projectId,+ tableName,+ query,+ cutoffCreatedAt,+ targetId,+ type,+ userId,+ } = batchActionEvent;++ logger.info(+ `Processing batch action ${actionId} in project ${projectId} requested by user ${userId ?? "unknown"}`,+ { projectId, actionId, actorType: "USER", userId },+ );if (type === BatchActionType.Create && !targetId) {throw new Error(`Target ID is required for create action`);@@ -280,6 +304,7 @@ export const handleBatchActionJob = async (batch.map((r) => r.id),projectId,targetId,+ userId,);}} else if (actionId === "eval-create") {diff --git a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts b/worker/src/features/batchAction/processTraceDeleteBatchAction.tsindex 2a0c94f7fa..ae1b59b034 100644--- a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts+++ b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts@@ -389,12 +389,18 @@ export const processTraceDeleteBatchAction = async ({});}- logger.info("Processing trace delete batch action page", {- batchActionId,- projectId: batchAction.projectId,- source: state.config.source,- traceCount: inFlightBatch.traceIds.length,- });+ logger.info(+ `Processing trace delete batch action page requested by user ${batchAction.userId}`,+ {+ batchActionId,+ projectId: batchAction.projectId,+ source: state.config.source,+ traceCount: inFlightBatch.traceIds.length,+ traceIds: inFlightBatch.traceIds,+ actorType: "USER",+ userId: batchAction.userId,+ },+ );if (await shouldSkipDeletion(diff --git a/worker/src/features/trace-delete-batch-action-runner/index.ts b/worker/src/features/trace-delete-batch-action-runner/index.tsindex af00ab945c..bd7ab4b3b5 100644--- a/worker/src/features/trace-delete-batch-action-runner/index.ts+++ b/worker/src/features/trace-delete-batch-action-runner/index.ts@@ -33,6 +33,7 @@ const ACTIVE_TRACE_DELETE_BATCH_ACTION_WHERE = {type ActiveTraceDeleteBatchAction = {id: string;projectId: string;+ userId: string;status: string;processedCount: number | null;failedCount: number | null;@@ -137,6 +138,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {logger.info(`${this.instanceName}: Processing trace delete action`, {batchActionId: batchAction.id,projectId: batchAction.projectId,+ userId: batchAction.userId,status: batchAction.status,source,activeActionCount,@@ -195,6 +197,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {select: {id: true,projectId: true,+ userId: true,status: true,processedCount: true,failedCount: true,diff --git a/worker/src/queues/traceDelete.ts b/worker/src/queues/traceDelete.tsindex 8cc456f654..29b89e05f3 100644--- a/worker/src/queues/traceDelete.ts+++ b/worker/src/queues/traceDelete.ts@@ -1,5 +1,7 @@import { Job, Processor } from "bullmq";import {+ formatActor,+ getActorLogMetadata,getCurrentSpan,logger,QueueName,@@ -21,6 +23,22 @@ export const traceDeleteProcessor: Processor = async (? job.data.payload.traceIds: [job.data.payload.traceId];+ const actor =+ "actor" in job.data.payload ? job.data.payload.actor : undefined;++ if (actor) {+ // The job may also pick up pending deletions requested by other actors;+ // the actor applies to the trace ids from this event only.+ logger.info(+ `Trace deletion job for ${eventTraceIds.length} traces in project ${projectId} requested by ${formatActor(actor)}`,+ {+ projectId,+ traceIds: eventTraceIds,+ actor: getActorLogMetadata(actor),+ },+ );+ }+const span = getCurrentSpan();const [toBeDeletedTraces, pendingEventTraceIds] = await Promise.all([