🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(audit-logs): relay staged audit events into ClickHouse (#18713)
Steffen Reuther4a72ba83ec8c
diff --git a/.env.prod.example b/.env.prod.exampleindex b7b29962c7..f932e756bc 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -440,6 +440,13 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"# NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"+# Moves staged audit events from Postgres into ClickHouse. Runs only when the+# cloud region above is set. While it is disabled, staged events wait in+# Postgres.+# LANGFUSE_AUDIT_LOG_RELAY_ENABLED=true+# LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS=1000+# LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS=60+# LANGFUSE_TEAM_SLACK_WEBHOOK=# LANGFUSE_FEEDBACK_INTAKE_SLACK_WEBHOOK=# LANGFUSE_NEW_USER_SIGNUP_WEBHOOK=diff --git a/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sql b/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sqlnew file mode 100644index 0000000000..c93ae47fdc--- /dev/null+++ b/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sql@@ -0,0 +1,3 @@+-- AlterTable+ALTER TABLE "audit_log_staging" ADD COLUMN "failed_at" TIMESTAMP(3),+ADD COLUMN "last_error" TEXT;diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex e79f8cbfb1..dda40d1a22 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -1090,9 +1090,14 @@ model AuditLog {// A row is deleted once ClickHouse has acknowledged its insert, so the table// stays near-empty. `payload` holds an `AuditLogStagingPayload`.model AuditLogStaging {- id String @id @db.Uuid // UUIDv7, so id order follows creation time- createdAt DateTime @default(now()) @map("created_at")+ id String @id @db.Uuid // UUIDv7, so id order follows creation time+ createdAt DateTime @default(now()) @map("created_at")payload Json+ // Set when the relay could not map `payload` to a ClickHouse row. The row+ // stays and is retried after a delay, e.g. once a newer worker that knows+ // its schema is deployed.+ failedAt DateTime? @map("failed_at")+ lastError String? @map("last_error")@@map("audit_log_staging")}diff --git a/packages/shared/src/server/repositories/auditLogs.ts b/packages/shared/src/server/repositories/auditLogs.tsindex 1545fe41f8..fa7dca67fb 100644--- a/packages/shared/src/server/repositories/auditLogs.ts+++ b/packages/shared/src/server/repositories/auditLogs.ts@@ -46,12 +46,19 @@ export async function stageAuditLogEvents(/*** Maps a staging row to its ClickHouse `audit_logs` row. The staging row's* `created_at` is the event timestamp. Throws when the payload does not- * match `AuditLogStagingPayloadSchema`.+ * match `AuditLogStagingPayloadSchema` or carries fields it does not know,+ * which a newer version may have staged, so that no field is dropped.*/export function convertAuditLogStagingRowToClickhouse(row: Pick<AuditLogStaging, "id" | "createdAt" | "payload">,): AuditLogRecordInsertType {const event = AuditLogStagingPayloadSchema.parse(row.payload);+ const unknownFields = findDroppedFields(row.payload, event);+ if (unknownFields.length > 0) {+ throw new Error(+ `Staged audit event has unknown fields: ${unknownFields.join(", ")}`,+ );+ }const { actor } = event;return {@@ -84,3 +91,24 @@ export function convertAuditLogStagingRowToClickhouse(request_id: event.request?.requestId ?? "",};}++/**+ * Returns the dotted paths of object keys in `input` that are missing from+ * `parsed`, i.e. the keys a zod object schema stripped while parsing.+ */+function findDroppedFields(+ input: unknown,+ parsed: unknown,+ path = "",+): string[] {+ if (!isRecord(input) || !isRecord(parsed)) return [];+ return Object.keys(input).flatMap((key) => {+ const keyPath = path === "" ? key : `${path}.${key}`;+ return key in parsed+ ? findDroppedFields(input[key], parsed[key], keyPath)+ : [keyPath];+ });+}++const isRecord = (value: unknown): value is Record<string, unknown> =>+ typeof value === "object" && value !== null && !Array.isArray(value);diff --git a/scripts/ci/write-web-test-env.sh b/scripts/ci/write-web-test-env.shindex cd9735927b..e49d57c4ec 100755--- a/scripts/ci/write-web-test-env.sh+++ b/scripts/ci/write-web-test-env.sh@@ -18,4 +18,7 @@ readonly OUTPUT="${2:?usage: write-web-test-env.sh <env example> <output>}"echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"echo "LANGFUSE_ENABLE_SCORES_V3_API=true"+ # The started worker shares the test database, and its relay would delete+ # staged audit rows that server tests read back.+ echo "LANGFUSE_AUDIT_LOG_RELAY_ENABLED=false"} > "$OUTPUT"diff --git a/web/src/__tests__/server/audit-log-staging.servertest.ts b/web/src/__tests__/server/audit-log-staging.servertest.tsindex 8fecfc3306..3491ea2efe 100644--- a/web/src/__tests__/server/audit-log-staging.servertest.ts+++ b/web/src/__tests__/server/audit-log-staging.servertest.ts@@ -209,4 +209,24 @@ describe("audit log staging", () => {resource_refs: ["trace:trace-�-1"],});});++ it("rejects staged payloads with fields it does not know", () => {+ // A newer web version may stage a field before this version maps it.+ expect(() =>+ convertAuditLogStagingRowToClickhouse({+ id: randomUUID(),+ createdAt: new Date(),+ payload: {+ kind: "change",+ orgId: randomUUID(),+ actor: { type: "USER", userId: "user-1", sessionId: "session-1" },+ resourceType: "prompt",+ resourceRefs: [],+ action: "delete",+ outcome: "success",+ dataSubject: "subject-1",+ },+ }),+ ).toThrow("actor.sessionId, dataSubject");+ });});diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 9e4a647b3b..e70a91e61b 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -88,6 +88,12 @@summaries reduce it to that threshold or below. The client idle timeout mustaccount for these pauses. Drain accepted summaries even on read failure. Never flush a failedstream's partial final trace; completion covers the query window, not future arrivals.+- Audit log relay: `src/features/audit-log-relay/index.ts` moves Postgres+ `audit_log_staging` rows into ClickHouse `audit_logs`, deleting them only+ after ClickHouse acknowledged the insert. It starts only in cloud+ deployments (`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` set). Rows whose payload+ fails the staging schema stay with `failed_at` set and are retried after+ 10 minutes.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`- `ClickhouseWriter.getInstance()` queues JSON rows; `getNativeInstance()` queuesdiff --git a/worker/src/__tests__/auditLogRelay.test.ts b/worker/src/__tests__/auditLogRelay.test.tsnew file mode 100644index 0000000000..6253e0bdc8--- /dev/null+++ b/worker/src/__tests__/auditLogRelay.test.ts@@ -0,0 +1,233 @@+import { randomUUID } from "crypto";+import { v7 as uuidv7 } from "uuid";+import { beforeEach, describe, expect, it } from "vitest";+import { prisma } from "@langfuse/shared/src/db";+import {+ queryClickhouse,+ stageAuditLogEvents,+} from "@langfuse/shared/src/server";+import {+ AuditLogRelay,+ insertAuditLogRecords,+} from "../features/audit-log-relay";++const stageChangeEvents = (orgId: string, count: number) =>+ stageAuditLogEvents(+ Array.from({ length: count }, () => ({+ kind: "change" as const,+ orgId,+ projectId: "project-1",+ actor: { type: "USER" as const, userId: "user-1" },+ resourceType: "prompt",+ resourceId: "prompt-1",+ action: "update",+ before: { version: 1 },+ after: { version: 2 },+ })),+ );++const stagedIds = async (): Promise<string[]> =>+ (+ await prisma.auditLogStaging.findMany({+ select: { id: true },+ orderBy: { id: "asc" },+ })+ ).map((row) => row.id);++// FINAL applies the ReplacingMergeTree collapse that a read with+// LIMIT 1 BY id would.+const auditLogIds = async (orgId: string): Promise<string[]> =>+ (+ await queryClickhouse<{ id: string }>({+ query:+ "SELECT id FROM audit_logs FINAL WHERE org_id = {orgId: String} ORDER BY id",+ params: { orgId },+ })+ ).map((row) => row.id);++const sorted = (ids: string[]) => [...ids].sort();++describe("AuditLogRelay", () => {+ // The relay claims every staged row, not only the ones a test staged.+ beforeEach(async () => {+ await prisma.auditLogStaging.deleteMany();+ });++ it("deletes staged rows only after ClickHouse acknowledged the insert", async () => {+ const orgId = randomUUID();+ const ids = await stageChangeEvents(orgId, 3);+ let stagedDuringInsert: string[] = [];+ const relay = new AuditLogRelay({+ insertRecords: async (records, abortSignal) => {+ stagedDuringInsert = await stagedIds();+ await insertAuditLogRecords(records, abortSignal);+ },+ });++ await expect(relay.relayBatch()).resolves.toEqual({+ claimed: 3,+ relayed: 3,+ failed: 0,+ });++ expect(stagedDuringInsert).toEqual(sorted(ids));+ expect(await stagedIds()).toEqual([]);+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));+ });++ it("leaves the rows in place when the ClickHouse insert fails", async () => {+ const orgId = randomUUID();+ const ids = await stageChangeEvents(orgId, 2);+ const relay = new AuditLogRelay({+ insertRecords: async () => {+ throw new Error("ClickHouse unavailable");+ },+ });++ await expect(relay.relayBatch()).rejects.toThrow("ClickHouse unavailable");++ const rows = await prisma.auditLogStaging.findMany({+ where: { id: { in: ids } },+ });+ expect(rows).toHaveLength(2);+ expect(rows.map((row) => row.failedAt)).toEqual([null, null]);++ await expect(new AuditLogRelay().relayBatch()).resolves.toMatchObject({+ relayed: 2,+ });+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));+ });++ it("collapses rows that are inserted again after a lost acknowledgement", async () => {+ const orgId = randomUUID();+ const ids = await stageChangeEvents(orgId, 2);+ let inserts = 0;+ const lostAck = new AuditLogRelay({+ insertRecords: async (records, abortSignal) => {+ await insertAuditLogRecords(records, abortSignal);+ inserts += 1;+ throw new Error("connection reset");+ },+ });++ await expect(lostAck.relayBatch()).rejects.toThrow("connection reset");+ expect(await stagedIds()).toEqual(sorted(ids));++ const relay = new AuditLogRelay({+ insertRecords: async (records, abortSignal) => {+ await insertAuditLogRecords(records, abortSignal);+ inserts += 1;+ },+ });+ await expect(relay.relayBatch()).resolves.toMatchObject({ relayed: 2 });++ expect(inserts).toBe(2);+ expect(await stagedIds()).toEqual([]);+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));+ });++ it("never lets concurrent batches claim the same rows", async () => {+ const orgId = randomUUID();+ const ids = await stageChangeEvents(orgId, 6);+ const claimedByBatch: string[][] = [];+ let releaseInserts!: () => void;+ const bothBatchesClaimed = new Promise<void>((resolve) => {+ releaseInserts = resolve;+ });+ const relay = new AuditLogRelay({+ batchSize: 3,+ insertRecords: async (records, abortSignal) => {+ claimedByBatch.push(records.map((record) => record.id));+ if (claimedByBatch.length === 2) releaseInserts();+ // Holds the first batch's row locks until the second batch claimed.+ await Promise.race([+ bothBatchesClaimed,+ new Promise((resolve) => setTimeout(resolve, 5_000)),+ ]);+ await insertAuditLogRecords(records, abortSignal);+ },+ });++ const results = await Promise.all([relay.relayBatch(), relay.relayBatch()]);++ expect(results.map((result) => result.claimed)).toEqual([3, 3]);+ const [first, second] = claimedByBatch;+ expect(first.filter((id) => second.includes(id))).toEqual([]);+ expect(sorted([...first, ...second])).toEqual(sorted(ids));+ expect(await stagedIds()).toEqual([]);+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));+ });++ it("marks a row it cannot map as failed and retries it after a delay", async () => {+ const orgId = randomUUID();+ const [validId] = await stageChangeEvents(orgId, 1);+ const { payload } = await prisma.auditLogStaging.findUniqueOrThrow({+ where: { id: validId },+ });+ // A value that a newer web version may write before this worker knows it.+ const failingId = uuidv7();+ await prisma.auditLogStaging.create({+ data: {+ id: failingId,+ createdAt: new Date(),+ payload: { ...(payload as object), surface: "grpc" },+ },+ });+ const relay = new AuditLogRelay();+ const shiftFailedAtBack = () => prisma.$executeRaw`+ UPDATE audit_log_staging+ SET failed_at = failed_at - INTERVAL '11 minutes'+ WHERE id = ${failingId}::uuid+ `;++ await expect(relay.relayBatch()).resolves.toEqual({+ claimed: 2,+ relayed: 1,+ failed: 1,+ });+ const failed = await prisma.auditLogStaging.findUniqueOrThrow({+ where: { id: failingId },+ });+ expect(failed.failedAt).not.toBeNull();+ expect(failed.lastError).toContain("surface");+ expect(await auditLogIds(orgId)).toEqual([validId]);++ // Not claimed again before the retry delay has passed.+ await expect(relay.relayBatch()).resolves.toMatchObject({ claimed: 0 });++ await shiftFailedAtBack();+ await expect(relay.relayBatch()).resolves.toMatchObject({+ claimed: 1,+ failed: 1,+ });+ const retried = await prisma.auditLogStaging.findUniqueOrThrow({+ where: { id: failingId },+ });+ expect(retried.failedAt!.getTime()).toBeGreaterThan(+ failed.failedAt!.getTime(),+ );++ // Stands in for a worker that knows the newer schema.+ await prisma.auditLogStaging.update({+ where: { id: failingId },+ data: { payload },+ });+ await shiftFailedAtBack();+ await expect(relay.relayBatch()).resolves.toEqual({+ claimed: 1,+ relayed: 1,+ failed: 0,+ });+ expect(await stagedIds()).toEqual([]);+ expect(await auditLogIds(orgId)).toEqual(sorted([validId, failingId]));+ });++ it("runs again right away while batches come back full", async () => {+ await stageChangeEvents(randomUUID(), 3);+ const relay = new AuditLogRelay({ batchSize: 2 });++ await expect(relay.processBatch()).resolves.toBe(0);+ await expect(relay.processBatch()).resolves.toBeUndefined();+ expect(await stagedIds()).toEqual([]);+ });+});diff --git a/worker/src/app.ts b/worker/src/app.tsindex f5ae962ac3..ad27dc7e31 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -107,6 +107,7 @@ import { BatchProjectBlobCleaner } from "./features/batch-project-blob-cleaner";import { QueueMetricsRunner } from "./features/queue-metrics-runner";import { MonitorRunner } from "./features/monitor-runner";import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";+import { AuditLogRelay } from "./features/audit-log-relay";import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";import { InAppAgentIntegrityRunner } from "./features/in-app-agent-integrity-runner";import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";@@ -860,6 +861,17 @@ if (env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_ENABLED === "true") {deletedMaskCleaner.start();}+// Moves staged audit events from Postgres into ClickHouse audit_logs+export let auditLogRelay: AuditLogRelay | null = null;++if (+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && // Only in cloud deployments+ env.LANGFUSE_AUDIT_LOG_RELAY_ENABLED === "true"+) {+ auditLogRelay = new AuditLogRelay();+ auditLogRelay.start();+}+// Queue metrics background reporterexport let queueMetricsRunner: QueueMetricsRunner | null = null;diff --git a/worker/src/env.ts b/worker/src/env.tsindex d0122c2fbc..846178c0af 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -670,6 +670,21 @@ const EnvSchema = z.object({.enum(["true", "false"]).default("false"), // Use ON CLUSTER and clusterAllReplicas for cleaner operations+ // Audit log relay: moves audit_log_staging rows into ClickHouse audit_logs.+ // Runs only in cloud deployments (NEXT_PUBLIC_LANGFUSE_CLOUD_REGION set),+ // where it is on unless disabled here.+ LANGFUSE_AUDIT_LOG_RELAY_ENABLED: z.enum(["true", "false"]).default("true"),+ LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(1_000),+ LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS: z.coerce+ .number()+ .int()+ .positive()+ .default(60), // A batch must finish within half of it+// Media Retention Cleaner configuration (S3/PostgreSQL)LANGFUSE_MEDIA_RETENTION_CLEANER_ITEM_LIMIT: z.coerce.number()diff --git a/worker/src/features/audit-log-relay/index.ts b/worker/src/features/audit-log-relay/index.tsnew file mode 100644index 0000000000..bd37b6e4f7--- /dev/null+++ b/worker/src/features/audit-log-relay/index.ts@@ -0,0 +1,336 @@+import { Prisma, prisma } from "@langfuse/shared/src/db";+import {+ clickhouseClient,+ convertAuditLogStagingRowToClickhouse,+ logger,+ recordGauge,+ recordIncrement,+ type AuditLogRecordInsertType,+} from "@langfuse/shared/src/server";+import { env } from "../../env";+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";++const METRIC_PREFIX = "langfuse.audit_log_relay";++const AUDIT_LOG_RELAY_LOCK_KEY = "langfuse:audit-log-relay";++const DEFAULT_BATCH_SIZE = 5_000;++// A row whose payload could not be mapped is claimed again after this delay,+// so a payload written by a newer web version relays once the worker that+// knows its schema is deployed.+const FAILED_ROW_RETRY_DELAY = Prisma.sql`INTERVAL '10 minutes'`;++const MAX_FAILURE_BACKOFF_MS = 30_000;++// Counting the staging table scans it, so the gauges are refreshed at most+// this often while a backlog drains batch after batch, and the scan is cut+// off well within the lease.+const STAGING_METRICS_INTERVAL_MS = 10_000;+const STAGING_METRICS_STATEMENT_TIMEOUT_MS = 5_000;++const LAST_ERROR_MAX_LENGTH = 1_000;+const LOGGED_FAILURES_MAX = 10;++/**+ * Moves rows from the Postgres `audit_log_staging` outbox into the ClickHouse+ * `audit_logs` table. Each batch runs in one Postgres transaction: it locks+ * up to `batchSize` rows, inserts them into ClickHouse and deletes them only+ * after ClickHouse acknowledged the insert. Any failure rolls the transaction+ * back, so the rows stay and the next batch retries them. A retry can insert+ * a row a second time; `audit_logs` collapses rows by id.+ *+ * Concurrent batches claim disjoint rows (FOR UPDATE SKIP LOCKED), so the+ * Redis lease only keeps workers from competing for the same rows. When Redis+ * is unavailable, every worker relays.+ */+export class AuditLogRelay extends PeriodicExclusiveRunner {+ private readonly intervalMs: number;+ private readonly lockTtlSeconds: number;+ private readonly batchSize: number;+ private readonly transactionTimeoutMs: number;+ private readonly insertRecords: AuditLogRecordInserter;+ private consecutiveFailures = 0;+ private lastStagingMetricsAt = 0;++ protected get defaultIntervalMs(): number {+ return this.intervalMs;+ }++ constructor(+ opts: {+ intervalMs?: number;+ lockTtlSeconds?: number;+ batchSize?: number;+ insertRecords?: AuditLogRecordInserter;+ } = {},+ ) {+ const lockTtlSeconds =+ opts.lockTtlSeconds ?? env.LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS;++ super({+ name: "AuditLogRelay",+ metricName: "audit_log_relay",+ lockKey: AUDIT_LOG_RELAY_LOCK_KEY,+ lockTtlSeconds,+ onUnavailable: "proceed",+ });++ this.lockTtlSeconds = lockTtlSeconds;+ this.intervalMs =+ opts.intervalMs ?? env.LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS;+ this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;+ // A batch that runs out of time rolls back before the lease can expire+ // under it.+ this.transactionTimeoutMs = (lockTtlSeconds * 1000) / 2;+ this.insertRecords = opts.insertRecords ?? insertAuditLogRecords;+ }++ public override start(): void {+ logger.info(`Starting ${this.instanceName}`, {+ intervalMs: this.intervalMs,+ lockTtlSeconds: this.lockTtlSeconds,+ batchSize: this.batchSize,+ });+ super.start();+ }++ protected async execute(): Promise<number | void> {+ return await this.withLock(+ async () => {+ try {+ const result = await this.relayBatch();+ this.consecutiveFailures = 0;+ // A full batch means more rows are waiting, so run again right away.+ return result.claimed === this.batchSize ? 0 : undefined;+ } finally {+ await this.recordStagingMetrics();+ }+ },+ () => this.backOffAfterFailure(),+ );+ }++ /**+ * Relays one batch without taking the lease. Rejects, with every claimed+ * row left in place, when the ClickHouse insert or the transaction fails.+ */+ public async relayBatch(): Promise<AuditLogRelayBatchResult> {+ const outcome = await prisma.$transaction(+ (tx) => this.relayClaimedRows(tx),+ { timeout: this.transactionTimeoutMs },+ );+ this.reportBatch(outcome);+ return {+ claimed: outcome.claimed,+ relayed: outcome.relayed,+ failed: outcome.failures.length,+ };+ }++ private async relayClaimedRows(+ tx: Prisma.TransactionClient,+ ): Promise<BatchOutcome> {+ const rows = await claimStagingRows(tx, this.batchSize);+ const { records, failures } = mapStagingRows(rows);+ await this.relayRecords(tx, records);+ await markFailedRows(tx, failures);+ return { claimed: rows.length, relayed: records.length, failures };+ }++ /** Inserts the records into ClickHouse, then deletes them from staging. */+ private async relayRecords(+ tx: Prisma.TransactionClient,+ records: AuditLogRecordInsertType[],+ ): Promise<void> {+ if (records.length === 0) return;+ await this.insertRecords(+ records,+ AbortSignal.timeout(this.transactionTimeoutMs),+ );+ await tx.auditLogStaging.deleteMany({+ where: { id: { in: records.map((record) => record.id) } },+ });+ }++ /** Records metrics and logs for a committed batch. */+ private reportBatch(outcome: BatchOutcome): void {+ if (outcome.relayed > 0) {+ recordIncrement(`${METRIC_PREFIX}.rows_relayed`, outcome.relayed);+ }+ if (outcome.failures.length === 0) return;++ recordIncrement(`${METRIC_PREFIX}.rows_failed`, outcome.failures.length);+ // The payload can hold before/after state, so only ids and errors are+ // logged.+ logger.warn(+ `${this.instanceName}: Staged audit events could not be mapped, retrying later`,+ {+ count: outcome.failures.length,+ failures: outcome.failures.slice(0, LOGGED_FAILURES_MAX),+ },+ );+ }++ private backOffAfterFailure(): number {+ recordIncrement(`${METRIC_PREFIX}.relay_errors`, 1);+ this.consecutiveFailures += 1;+ return Math.min(+ this.intervalMs * 2 ** this.consecutiveFailures,+ MAX_FAILURE_BACKOFF_MS,+ );+ }++ private async recordStagingMetrics(): Promise<void> {+ const now = Date.now();+ if (now - this.lastStagingMetricsAt < STAGING_METRICS_INTERVAL_MS) {+ return;+ }+ this.lastStagingMetricsAt = now;++ // Metrics must not stop the relay, so a failed query is only logged.+ try {+ const metrics = await readStagingMetrics();+ recordGauge(`${METRIC_PREFIX}.staging_rows`, Number(metrics.pending));+ recordGauge(`${METRIC_PREFIX}.failed_rows`, Number(metrics.failed));+ recordGauge(+ `${METRIC_PREFIX}.oldest_row_age_seconds`,+ metrics.oldest_pending+ ? Math.max(+ Math.floor((now - metrics.oldest_pending.getTime()) / 1000),+ 0,+ )+ : 0,+ );+ } catch (error) {+ logger.error(`${this.instanceName}: Failed to read staging metrics`, {+ error,+ });+ }+ }+}++/**+ * Writes the batch as one synchronous insert, outside ClickhouseWriter, so a+ * resolved promise means ClickHouse has stored the rows. The relay already+ * batches, so an async insert would only hold the claimed rows until the+ * server-side buffer flushes.+ */+export async function insertAuditLogRecords(+ records: AuditLogRecordInsertType[],+ abortSignal: AbortSignal,+): Promise<void> {+ await clickhouseClient().insert({+ table: "audit_logs",+ values: records,+ format: "JSONEachRow",+ abort_signal: abortSignal,+ clickhouse_settings: { async_insert: 0 },+ });+}++/** Locks the oldest due rows; concurrent claims skip each other's rows. */+function claimStagingRows(+ tx: Prisma.TransactionClient,+ batchSize: number,+): Promise<ClaimedStagingRow[]> {+ return tx.$queryRaw<ClaimedStagingRow[]>`+ SELECT id, created_at, payload+ FROM audit_log_staging+ WHERE failed_at IS NULL+ OR failed_at < CURRENT_TIMESTAMP - ${FAILED_ROW_RETRY_DELAY}+ ORDER BY id+ LIMIT ${batchSize}+ FOR UPDATE SKIP LOCKED+ `;+}++function mapStagingRows(rows: ClaimedStagingRow[]): {+ records: AuditLogRecordInsertType[];+ failures: FailedStagingRow[];+} {+ const records: AuditLogRecordInsertType[] = [];+ const failures: FailedStagingRow[] = [];+ for (const row of rows) {+ try {+ records.push(+ convertAuditLogStagingRowToClickhouse({+ id: row.id,+ createdAt: row.created_at,+ payload: row.payload,+ }),+ );+ } catch (error) {+ failures.push({ id: row.id, error: describeError(error) });+ }+ }+ return { records, failures };+}++async function markFailedRows(+ tx: Prisma.TransactionClient,+ failures: FailedStagingRow[],+): Promise<void> {+ if (failures.length === 0) return;+ await tx.$executeRaw`+ UPDATE audit_log_staging AS staging+ SET failed_at = CURRENT_TIMESTAMP, last_error = failure.error+ FROM unnest(+ ${failures.map((failure) => failure.id)}::uuid[],+ ${failures.map((failure) => failure.error)}::text[]+ ) AS failure(id, error)+ WHERE staging.id = failure.id+ `;+}++async function readStagingMetrics(): Promise<StagingMetricsRow> {+ const [, [metrics]] = await prisma.$transaction([+ prisma.$queryRaw`SELECT set_config('statement_timeout', ${String(STAGING_METRICS_STATEMENT_TIMEOUT_MS)}, true)`,+ prisma.$queryRaw<StagingMetricsRow[]>`+ SELECT+ count(*) FILTER (WHERE failed_at IS NULL) AS pending,+ count(*) FILTER (WHERE failed_at IS NOT NULL) AS failed,+ min(created_at) FILTER (WHERE failed_at IS NULL) AS oldest_pending+ FROM audit_log_staging+ `,+ ]);+ return metrics;+}++function describeError(error: unknown): string {+ return (error instanceof Error ? error.message : String(error)).slice(+ 0,+ LAST_ERROR_MAX_LENGTH,+ );+}++export type AuditLogRecordInserter = (+ records: AuditLogRecordInsertType[],+ abortSignal: AbortSignal,+) => Promise<void>;++export type AuditLogRelayBatchResult = {+ claimed: number;+ relayed: number;+ failed: number;+};++type ClaimedStagingRow = {+ id: string;+ created_at: Date;+ payload: Prisma.JsonValue;+};++type FailedStagingRow = { id: string; error: string };++type BatchOutcome = {+ claimed: number;+ relayed: number;+ failures: FailedStagingRow[];+};++type StagingMetricsRow = {+ pending: bigint;+ failed: bigint;+ oldest_pending: Date | null;+};diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 149ee32c02..150510c4bf 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -21,6 +21,7 @@ import {traceDeleteBatchActionRunner,inAppAgentIntegrityRunner,deletedMaskCleaner,+ auditLogRelay,queueMetricsRunner,monitorRunners,inAppAgentDlqRetryRunner,@@ -85,6 +86,8 @@ const runDrainAndClose = async () => {// Stop deleted-mask cleanerdeletedMaskCleaner?.stop();+ auditLogRelay?.stop();+// Stop queue metrics runnerqueueMetricsRunner?.stop();traceBatchMetricsRunner?.stop();