Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(audit-logs): relay staged audit events into ClickHouse (#18713)

Steffen Reuther4a72ba83ec8c
diff --git a/.env.prod.example b/.env.prod.example
index b7b29962c7..f932e756bc 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -440,6 +440,13 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
# NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
+# Moves staged audit events from Postgres into ClickHouse. Runs only when the
+# cloud region above is set. While it is disabled, staged events wait in
+# Postgres.
+# LANGFUSE_AUDIT_LOG_RELAY_ENABLED=true
+# LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS=1000
+# LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS=60
+
# LANGFUSE_TEAM_SLACK_WEBHOOK=
# LANGFUSE_FEEDBACK_INTAKE_SLACK_WEBHOOK=
# LANGFUSE_NEW_USER_SIGNUP_WEBHOOK=
diff --git a/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sql b/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sql
new file mode 100644
index 0000000000..c93ae47fdc
--- /dev/null
+++ b/packages/shared/prisma/migrations/20261013123042_add_audit_log_staging_failure_columns/migration.sql
@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "audit_log_staging" ADD COLUMN "failed_at" TIMESTAMP(3),
+ADD COLUMN "last_error" TEXT;
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index e79f8cbfb1..dda40d1a22 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -1090,9 +1090,14 @@ model AuditLog {
// A row is deleted once ClickHouse has acknowledged its insert, so the table
// stays near-empty. `payload` holds an `AuditLogStagingPayload`.
model AuditLogStaging {
- id String @id @db.Uuid // UUIDv7, so id order follows creation time
- createdAt DateTime @default(now()) @map("created_at")
+ id String @id @db.Uuid // UUIDv7, so id order follows creation time
+ createdAt DateTime @default(now()) @map("created_at")
payload Json
+ // Set when the relay could not map `payload` to a ClickHouse row. The row
+ // stays and is retried after a delay, e.g. once a newer worker that knows
+ // its schema is deployed.
+ failedAt DateTime? @map("failed_at")
+ lastError String? @map("last_error")
@@map("audit_log_staging")
}
diff --git a/packages/shared/src/server/repositories/auditLogs.ts b/packages/shared/src/server/repositories/auditLogs.ts
index 1545fe41f8..fa7dca67fb 100644
--- a/packages/shared/src/server/repositories/auditLogs.ts
+++ b/packages/shared/src/server/repositories/auditLogs.ts
@@ -46,12 +46,19 @@ export async function stageAuditLogEvents(
/**
* Maps a staging row to its ClickHouse `audit_logs` row. The staging row's
* `created_at` is the event timestamp. Throws when the payload does not
- * match `AuditLogStagingPayloadSchema`.
+ * match `AuditLogStagingPayloadSchema` or carries fields it does not know,
+ * which a newer version may have staged, so that no field is dropped.
*/
export function convertAuditLogStagingRowToClickhouse(
row: Pick<AuditLogStaging, "id" | "createdAt" | "payload">,
): AuditLogRecordInsertType {
const event = AuditLogStagingPayloadSchema.parse(row.payload);
+ const unknownFields = findDroppedFields(row.payload, event);
+ if (unknownFields.length > 0) {
+ throw new Error(
+ `Staged audit event has unknown fields: ${unknownFields.join(", ")}`,
+ );
+ }
const { actor } = event;
return {
@@ -84,3 +91,24 @@ export function convertAuditLogStagingRowToClickhouse(
request_id: event.request?.requestId ?? "",
};
}
+
+/**
+ * Returns the dotted paths of object keys in `input` that are missing from
+ * `parsed`, i.e. the keys a zod object schema stripped while parsing.
+ */
+function findDroppedFields(
+ input: unknown,
+ parsed: unknown,
+ path = "",
+): string[] {
+ if (!isRecord(input) || !isRecord(parsed)) return [];
+ return Object.keys(input).flatMap((key) => {
+ const keyPath = path === "" ? key : `${path}.${key}`;
+ return key in parsed
+ ? findDroppedFields(input[key], parsed[key], keyPath)
+ : [keyPath];
+ });
+}
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+ typeof value === "object" && value !== null && !Array.isArray(value);
diff --git a/scripts/ci/write-web-test-env.sh b/scripts/ci/write-web-test-env.sh
index cd9735927b..e49d57c4ec 100755
--- a/scripts/ci/write-web-test-env.sh
+++ b/scripts/ci/write-web-test-env.sh
@@ -18,4 +18,7 @@ readonly OUTPUT="${2:?usage: write-web-test-env.sh <env example> <output>}"
echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"
echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"
echo "LANGFUSE_ENABLE_SCORES_V3_API=true"
+ # The started worker shares the test database, and its relay would delete
+ # staged audit rows that server tests read back.
+ echo "LANGFUSE_AUDIT_LOG_RELAY_ENABLED=false"
} > "$OUTPUT"
diff --git a/web/src/__tests__/server/audit-log-staging.servertest.ts b/web/src/__tests__/server/audit-log-staging.servertest.ts
index 8fecfc3306..3491ea2efe 100644
--- a/web/src/__tests__/server/audit-log-staging.servertest.ts
+++ b/web/src/__tests__/server/audit-log-staging.servertest.ts
@@ -209,4 +209,24 @@ describe("audit log staging", () => {
resource_refs: ["trace:trace-�-1"],
});
});
+
+ it("rejects staged payloads with fields it does not know", () => {
+ // A newer web version may stage a field before this version maps it.
+ expect(() =>
+ convertAuditLogStagingRowToClickhouse({
+ id: randomUUID(),
+ createdAt: new Date(),
+ payload: {
+ kind: "change",
+ orgId: randomUUID(),
+ actor: { type: "USER", userId: "user-1", sessionId: "session-1" },
+ resourceType: "prompt",
+ resourceRefs: [],
+ action: "delete",
+ outcome: "success",
+ dataSubject: "subject-1",
+ },
+ }),
+ ).toThrow("actor.sessionId, dataSubject");
+ });
});
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 9e4a647b3b..e70a91e61b 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -88,6 +88,12 @@
summaries reduce it to that threshold or below. The client idle timeout must
account for these pauses. Drain accepted summaries even on read failure. Never flush a failed
stream's partial final trace; completion covers the query window, not future arrivals.
+- Audit log relay: `src/features/audit-log-relay/index.ts` moves Postgres
+ `audit_log_staging` rows into ClickHouse `audit_logs`, deleting them only
+ after ClickHouse acknowledged the insert. It starts only in cloud
+ deployments (`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` set). Rows whose payload
+ fails the staging schema stay with `failed_at` set and are retried after
+ 10 minutes.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
- `ClickhouseWriter.getInstance()` queues JSON rows; `getNativeInstance()` queues
diff --git a/worker/src/__tests__/auditLogRelay.test.ts b/worker/src/__tests__/auditLogRelay.test.ts
new file mode 100644
index 0000000000..6253e0bdc8
--- /dev/null
+++ b/worker/src/__tests__/auditLogRelay.test.ts
@@ -0,0 +1,233 @@
+import { randomUUID } from "crypto";
+import { v7 as uuidv7 } from "uuid";
+import { beforeEach, describe, expect, it } from "vitest";
+import { prisma } from "@langfuse/shared/src/db";
+import {
+ queryClickhouse,
+ stageAuditLogEvents,
+} from "@langfuse/shared/src/server";
+import {
+ AuditLogRelay,
+ insertAuditLogRecords,
+} from "../features/audit-log-relay";
+
+const stageChangeEvents = (orgId: string, count: number) =>
+ stageAuditLogEvents(
+ Array.from({ length: count }, () => ({
+ kind: "change" as const,
+ orgId,
+ projectId: "project-1",
+ actor: { type: "USER" as const, userId: "user-1" },
+ resourceType: "prompt",
+ resourceId: "prompt-1",
+ action: "update",
+ before: { version: 1 },
+ after: { version: 2 },
+ })),
+ );
+
+const stagedIds = async (): Promise<string[]> =>
+ (
+ await prisma.auditLogStaging.findMany({
+ select: { id: true },
+ orderBy: { id: "asc" },
+ })
+ ).map((row) => row.id);
+
+// FINAL applies the ReplacingMergeTree collapse that a read with
+// LIMIT 1 BY id would.
+const auditLogIds = async (orgId: string): Promise<string[]> =>
+ (
+ await queryClickhouse<{ id: string }>({
+ query:
+ "SELECT id FROM audit_logs FINAL WHERE org_id = {orgId: String} ORDER BY id",
+ params: { orgId },
+ })
+ ).map((row) => row.id);
+
+const sorted = (ids: string[]) => [...ids].sort();
+
+describe("AuditLogRelay", () => {
+ // The relay claims every staged row, not only the ones a test staged.
+ beforeEach(async () => {
+ await prisma.auditLogStaging.deleteMany();
+ });
+
+ it("deletes staged rows only after ClickHouse acknowledged the insert", async () => {
+ const orgId = randomUUID();
+ const ids = await stageChangeEvents(orgId, 3);
+ let stagedDuringInsert: string[] = [];
+ const relay = new AuditLogRelay({
+ insertRecords: async (records, abortSignal) => {
+ stagedDuringInsert = await stagedIds();
+ await insertAuditLogRecords(records, abortSignal);
+ },
+ });
+
+ await expect(relay.relayBatch()).resolves.toEqual({
+ claimed: 3,
+ relayed: 3,
+ failed: 0,
+ });
+
+ expect(stagedDuringInsert).toEqual(sorted(ids));
+ expect(await stagedIds()).toEqual([]);
+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));
+ });
+
+ it("leaves the rows in place when the ClickHouse insert fails", async () => {
+ const orgId = randomUUID();
+ const ids = await stageChangeEvents(orgId, 2);
+ const relay = new AuditLogRelay({
+ insertRecords: async () => {
+ throw new Error("ClickHouse unavailable");
+ },
+ });
+
+ await expect(relay.relayBatch()).rejects.toThrow("ClickHouse unavailable");
+
+ const rows = await prisma.auditLogStaging.findMany({
+ where: { id: { in: ids } },
+ });
+ expect(rows).toHaveLength(2);
+ expect(rows.map((row) => row.failedAt)).toEqual([null, null]);
+
+ await expect(new AuditLogRelay().relayBatch()).resolves.toMatchObject({
+ relayed: 2,
+ });
+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));
+ });
+
+ it("collapses rows that are inserted again after a lost acknowledgement", async () => {
+ const orgId = randomUUID();
+ const ids = await stageChangeEvents(orgId, 2);
+ let inserts = 0;
+ const lostAck = new AuditLogRelay({
+ insertRecords: async (records, abortSignal) => {
+ await insertAuditLogRecords(records, abortSignal);
+ inserts += 1;
+ throw new Error("connection reset");
+ },
+ });
+
+ await expect(lostAck.relayBatch()).rejects.toThrow("connection reset");
+ expect(await stagedIds()).toEqual(sorted(ids));
+
+ const relay = new AuditLogRelay({
+ insertRecords: async (records, abortSignal) => {
+ await insertAuditLogRecords(records, abortSignal);
+ inserts += 1;
+ },
+ });
+ await expect(relay.relayBatch()).resolves.toMatchObject({ relayed: 2 });
+
+ expect(inserts).toBe(2);
+ expect(await stagedIds()).toEqual([]);
+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));
+ });
+
+ it("never lets concurrent batches claim the same rows", async () => {
+ const orgId = randomUUID();
+ const ids = await stageChangeEvents(orgId, 6);
+ const claimedByBatch: string[][] = [];
+ let releaseInserts!: () => void;
+ const bothBatchesClaimed = new Promise<void>((resolve) => {
+ releaseInserts = resolve;
+ });
+ const relay = new AuditLogRelay({
+ batchSize: 3,
+ insertRecords: async (records, abortSignal) => {
+ claimedByBatch.push(records.map((record) => record.id));
+ if (claimedByBatch.length === 2) releaseInserts();
+ // Holds the first batch's row locks until the second batch claimed.
+ await Promise.race([
+ bothBatchesClaimed,
+ new Promise((resolve) => setTimeout(resolve, 5_000)),
+ ]);
+ await insertAuditLogRecords(records, abortSignal);
+ },
+ });
+
+ const results = await Promise.all([relay.relayBatch(), relay.relayBatch()]);
+
+ expect(results.map((result) => result.claimed)).toEqual([3, 3]);
+ const [first, second] = claimedByBatch;
+ expect(first.filter((id) => second.includes(id))).toEqual([]);
+ expect(sorted([...first, ...second])).toEqual(sorted(ids));
+ expect(await stagedIds()).toEqual([]);
+ expect(await auditLogIds(orgId)).toEqual(sorted(ids));
+ });
+
+ it("marks a row it cannot map as failed and retries it after a delay", async () => {
+ const orgId = randomUUID();
+ const [validId] = await stageChangeEvents(orgId, 1);
+ const { payload } = await prisma.auditLogStaging.findUniqueOrThrow({
+ where: { id: validId },
+ });
+ // A value that a newer web version may write before this worker knows it.
+ const failingId = uuidv7();
+ await prisma.auditLogStaging.create({
+ data: {
+ id: failingId,
+ createdAt: new Date(),
+ payload: { ...(payload as object), surface: "grpc" },
+ },
+ });
+ const relay = new AuditLogRelay();
+ const shiftFailedAtBack = () => prisma.$executeRaw`
+ UPDATE audit_log_staging
+ SET failed_at = failed_at - INTERVAL '11 minutes'
+ WHERE id = ${failingId}::uuid
+ `;
+
+ await expect(relay.relayBatch()).resolves.toEqual({
+ claimed: 2,
+ relayed: 1,
+ failed: 1,
+ });
+ const failed = await prisma.auditLogStaging.findUniqueOrThrow({
+ where: { id: failingId },
+ });
+ expect(failed.failedAt).not.toBeNull();
+ expect(failed.lastError).toContain("surface");
+ expect(await auditLogIds(orgId)).toEqual([validId]);
+
+ // Not claimed again before the retry delay has passed.
+ await expect(relay.relayBatch()).resolves.toMatchObject({ claimed: 0 });
+
+ await shiftFailedAtBack();
+ await expect(relay.relayBatch()).resolves.toMatchObject({
+ claimed: 1,
+ failed: 1,
+ });
+ const retried = await prisma.auditLogStaging.findUniqueOrThrow({
+ where: { id: failingId },
+ });
+ expect(retried.failedAt!.getTime()).toBeGreaterThan(
+ failed.failedAt!.getTime(),
+ );
+
+ // Stands in for a worker that knows the newer schema.
+ await prisma.auditLogStaging.update({
+ where: { id: failingId },
+ data: { payload },
+ });
+ await shiftFailedAtBack();
+ await expect(relay.relayBatch()).resolves.toEqual({
+ claimed: 1,
+ relayed: 1,
+ failed: 0,
+ });
+ expect(await stagedIds()).toEqual([]);
+ expect(await auditLogIds(orgId)).toEqual(sorted([validId, failingId]));
+ });
+
+ it("runs again right away while batches come back full", async () => {
+ await stageChangeEvents(randomUUID(), 3);
+ const relay = new AuditLogRelay({ batchSize: 2 });
+
+ await expect(relay.processBatch()).resolves.toBe(0);
+ await expect(relay.processBatch()).resolves.toBeUndefined();
+ expect(await stagedIds()).toEqual([]);
+ });
+});
diff --git a/worker/src/app.ts b/worker/src/app.ts
index f5ae962ac3..ad27dc7e31 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -107,6 +107,7 @@ import { BatchProjectBlobCleaner } from "./features/batch-project-blob-cleaner";
import { QueueMetricsRunner } from "./features/queue-metrics-runner";
import { MonitorRunner } from "./features/monitor-runner";
import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";
+import { AuditLogRelay } from "./features/audit-log-relay";
import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";
import { InAppAgentIntegrityRunner } from "./features/in-app-agent-integrity-runner";
import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";
@@ -860,6 +861,17 @@ if (env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_ENABLED === "true") {
deletedMaskCleaner.start();
}
+// Moves staged audit events from Postgres into ClickHouse audit_logs
+export let auditLogRelay: AuditLogRelay | null = null;
+
+if (
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && // Only in cloud deployments
+ env.LANGFUSE_AUDIT_LOG_RELAY_ENABLED === "true"
+) {
+ auditLogRelay = new AuditLogRelay();
+ auditLogRelay.start();
+}
+
// Queue metrics background reporter
export let queueMetricsRunner: QueueMetricsRunner | null = null;
diff --git a/worker/src/env.ts b/worker/src/env.ts
index d0122c2fbc..846178c0af 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -670,6 +670,21 @@ const EnvSchema = z.object({
.enum(["true", "false"])
.default("false"), // Use ON CLUSTER and clusterAllReplicas for cleaner operations
+ // Audit log relay: moves audit_log_staging rows into ClickHouse audit_logs.
+ // Runs only in cloud deployments (NEXT_PUBLIC_LANGFUSE_CLOUD_REGION set),
+ // where it is on unless disabled here.
+ LANGFUSE_AUDIT_LOG_RELAY_ENABLED: z.enum(["true", "false"]).default("true"),
+ LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(1_000),
+ LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(60), // A batch must finish within half of it
+
// Media Retention Cleaner configuration (S3/PostgreSQL)
LANGFUSE_MEDIA_RETENTION_CLEANER_ITEM_LIMIT: z.coerce
.number()
diff --git a/worker/src/features/audit-log-relay/index.ts b/worker/src/features/audit-log-relay/index.ts
new file mode 100644
index 0000000000..bd37b6e4f7
--- /dev/null
+++ b/worker/src/features/audit-log-relay/index.ts
@@ -0,0 +1,336 @@
+import { Prisma, prisma } from "@langfuse/shared/src/db";
+import {
+ clickhouseClient,
+ convertAuditLogStagingRowToClickhouse,
+ logger,
+ recordGauge,
+ recordIncrement,
+ type AuditLogRecordInsertType,
+} from "@langfuse/shared/src/server";
+import { env } from "../../env";
+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+
+const METRIC_PREFIX = "langfuse.audit_log_relay";
+
+const AUDIT_LOG_RELAY_LOCK_KEY = "langfuse:audit-log-relay";
+
+const DEFAULT_BATCH_SIZE = 5_000;
+
+// A row whose payload could not be mapped is claimed again after this delay,
+// so a payload written by a newer web version relays once the worker that
+// knows its schema is deployed.
+const FAILED_ROW_RETRY_DELAY = Prisma.sql`INTERVAL '10 minutes'`;
+
+const MAX_FAILURE_BACKOFF_MS = 30_000;
+
+// Counting the staging table scans it, so the gauges are refreshed at most
+// this often while a backlog drains batch after batch, and the scan is cut
+// off well within the lease.
+const STAGING_METRICS_INTERVAL_MS = 10_000;
+const STAGING_METRICS_STATEMENT_TIMEOUT_MS = 5_000;
+
+const LAST_ERROR_MAX_LENGTH = 1_000;
+const LOGGED_FAILURES_MAX = 10;
+
+/**
+ * Moves rows from the Postgres `audit_log_staging` outbox into the ClickHouse
+ * `audit_logs` table. Each batch runs in one Postgres transaction: it locks
+ * up to `batchSize` rows, inserts them into ClickHouse and deletes them only
+ * after ClickHouse acknowledged the insert. Any failure rolls the transaction
+ * back, so the rows stay and the next batch retries them. A retry can insert
+ * a row a second time; `audit_logs` collapses rows by id.
+ *
+ * Concurrent batches claim disjoint rows (FOR UPDATE SKIP LOCKED), so the
+ * Redis lease only keeps workers from competing for the same rows. When Redis
+ * is unavailable, every worker relays.
+ */
+export class AuditLogRelay extends PeriodicExclusiveRunner {
+ private readonly intervalMs: number;
+ private readonly lockTtlSeconds: number;
+ private readonly batchSize: number;
+ private readonly transactionTimeoutMs: number;
+ private readonly insertRecords: AuditLogRecordInserter;
+ private consecutiveFailures = 0;
+ private lastStagingMetricsAt = 0;
+
+ protected get defaultIntervalMs(): number {
+ return this.intervalMs;
+ }
+
+ constructor(
+ opts: {
+ intervalMs?: number;
+ lockTtlSeconds?: number;
+ batchSize?: number;
+ insertRecords?: AuditLogRecordInserter;
+ } = {},
+ ) {
+ const lockTtlSeconds =
+ opts.lockTtlSeconds ?? env.LANGFUSE_AUDIT_LOG_RELAY_LOCK_TTL_SECONDS;
+
+ super({
+ name: "AuditLogRelay",
+ metricName: "audit_log_relay",
+ lockKey: AUDIT_LOG_RELAY_LOCK_KEY,
+ lockTtlSeconds,
+ onUnavailable: "proceed",
+ });
+
+ this.lockTtlSeconds = lockTtlSeconds;
+ this.intervalMs =
+ opts.intervalMs ?? env.LANGFUSE_AUDIT_LOG_RELAY_INTERVAL_MS;
+ this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+ // A batch that runs out of time rolls back before the lease can expire
+ // under it.
+ this.transactionTimeoutMs = (lockTtlSeconds * 1000) / 2;
+ this.insertRecords = opts.insertRecords ?? insertAuditLogRecords;
+ }
+
+ public override start(): void {
+ logger.info(`Starting ${this.instanceName}`, {
+ intervalMs: this.intervalMs,
+ lockTtlSeconds: this.lockTtlSeconds,
+ batchSize: this.batchSize,
+ });
+ super.start();
+ }
+
+ protected async execute(): Promise<number | void> {
+ return await this.withLock(
+ async () => {
+ try {
+ const result = await this.relayBatch();
+ this.consecutiveFailures = 0;
+ // A full batch means more rows are waiting, so run again right away.
+ return result.claimed === this.batchSize ? 0 : undefined;
+ } finally {
+ await this.recordStagingMetrics();
+ }
+ },
+ () => this.backOffAfterFailure(),
+ );
+ }
+
+ /**
+ * Relays one batch without taking the lease. Rejects, with every claimed
+ * row left in place, when the ClickHouse insert or the transaction fails.
+ */
+ public async relayBatch(): Promise<AuditLogRelayBatchResult> {
+ const outcome = await prisma.$transaction(
+ (tx) => this.relayClaimedRows(tx),
+ { timeout: this.transactionTimeoutMs },
+ );
+ this.reportBatch(outcome);
+ return {
+ claimed: outcome.claimed,
+ relayed: outcome.relayed,
+ failed: outcome.failures.length,
+ };
+ }
+
+ private async relayClaimedRows(
+ tx: Prisma.TransactionClient,
+ ): Promise<BatchOutcome> {
+ const rows = await claimStagingRows(tx, this.batchSize);
+ const { records, failures } = mapStagingRows(rows);
+ await this.relayRecords(tx, records);
+ await markFailedRows(tx, failures);
+ return { claimed: rows.length, relayed: records.length, failures };
+ }
+
+ /** Inserts the records into ClickHouse, then deletes them from staging. */
+ private async relayRecords(
+ tx: Prisma.TransactionClient,
+ records: AuditLogRecordInsertType[],
+ ): Promise<void> {
+ if (records.length === 0) return;
+ await this.insertRecords(
+ records,
+ AbortSignal.timeout(this.transactionTimeoutMs),
+ );
+ await tx.auditLogStaging.deleteMany({
+ where: { id: { in: records.map((record) => record.id) } },
+ });
+ }
+
+ /** Records metrics and logs for a committed batch. */
+ private reportBatch(outcome: BatchOutcome): void {
+ if (outcome.relayed > 0) {
+ recordIncrement(`${METRIC_PREFIX}.rows_relayed`, outcome.relayed);
+ }
+ if (outcome.failures.length === 0) return;
+
+ recordIncrement(`${METRIC_PREFIX}.rows_failed`, outcome.failures.length);
+ // The payload can hold before/after state, so only ids and errors are
+ // logged.
+ logger.warn(
+ `${this.instanceName}: Staged audit events could not be mapped, retrying later`,
+ {
+ count: outcome.failures.length,
+ failures: outcome.failures.slice(0, LOGGED_FAILURES_MAX),
+ },
+ );
+ }
+
+ private backOffAfterFailure(): number {
+ recordIncrement(`${METRIC_PREFIX}.relay_errors`, 1);
+ this.consecutiveFailures += 1;
+ return Math.min(
+ this.intervalMs * 2 ** this.consecutiveFailures,
+ MAX_FAILURE_BACKOFF_MS,
+ );
+ }
+
+ private async recordStagingMetrics(): Promise<void> {
+ const now = Date.now();
+ if (now - this.lastStagingMetricsAt < STAGING_METRICS_INTERVAL_MS) {
+ return;
+ }
+ this.lastStagingMetricsAt = now;
+
+ // Metrics must not stop the relay, so a failed query is only logged.
+ try {
+ const metrics = await readStagingMetrics();
+ recordGauge(`${METRIC_PREFIX}.staging_rows`, Number(metrics.pending));
+ recordGauge(`${METRIC_PREFIX}.failed_rows`, Number(metrics.failed));
+ recordGauge(
+ `${METRIC_PREFIX}.oldest_row_age_seconds`,
+ metrics.oldest_pending
+ ? Math.max(
+ Math.floor((now - metrics.oldest_pending.getTime()) / 1000),
+ 0,
+ )
+ : 0,
+ );
+ } catch (error) {
+ logger.error(`${this.instanceName}: Failed to read staging metrics`, {
+ error,
+ });
+ }
+ }
+}
+
+/**
+ * Writes the batch as one synchronous insert, outside ClickhouseWriter, so a
+ * resolved promise means ClickHouse has stored the rows. The relay already
+ * batches, so an async insert would only hold the claimed rows until the
+ * server-side buffer flushes.
+ */
+export async function insertAuditLogRecords(
+ records: AuditLogRecordInsertType[],
+ abortSignal: AbortSignal,
+): Promise<void> {
+ await clickhouseClient().insert({
+ table: "audit_logs",
+ values: records,
+ format: "JSONEachRow",
+ abort_signal: abortSignal,
+ clickhouse_settings: { async_insert: 0 },
+ });
+}
+
+/** Locks the oldest due rows; concurrent claims skip each other's rows. */
+function claimStagingRows(
+ tx: Prisma.TransactionClient,
+ batchSize: number,
+): Promise<ClaimedStagingRow[]> {
+ return tx.$queryRaw<ClaimedStagingRow[]>`
+ SELECT id, created_at, payload
+ FROM audit_log_staging
+ WHERE failed_at IS NULL
+ OR failed_at < CURRENT_TIMESTAMP - ${FAILED_ROW_RETRY_DELAY}
+ ORDER BY id
+ LIMIT ${batchSize}
+ FOR UPDATE SKIP LOCKED
+ `;
+}
+
+function mapStagingRows(rows: ClaimedStagingRow[]): {
+ records: AuditLogRecordInsertType[];
+ failures: FailedStagingRow[];
+} {
+ const records: AuditLogRecordInsertType[] = [];
+ const failures: FailedStagingRow[] = [];
+ for (const row of rows) {
+ try {
+ records.push(
+ convertAuditLogStagingRowToClickhouse({
+ id: row.id,
+ createdAt: row.created_at,
+ payload: row.payload,
+ }),
+ );
+ } catch (error) {
+ failures.push({ id: row.id, error: describeError(error) });
+ }
+ }
+ return { records, failures };
+}
+
+async function markFailedRows(
+ tx: Prisma.TransactionClient,
+ failures: FailedStagingRow[],
+): Promise<void> {
+ if (failures.length === 0) return;
+ await tx.$executeRaw`
+ UPDATE audit_log_staging AS staging
+ SET failed_at = CURRENT_TIMESTAMP, last_error = failure.error
+ FROM unnest(
+ ${failures.map((failure) => failure.id)}::uuid[],
+ ${failures.map((failure) => failure.error)}::text[]
+ ) AS failure(id, error)
+ WHERE staging.id = failure.id
+ `;
+}
+
+async function readStagingMetrics(): Promise<StagingMetricsRow> {
+ const [, [metrics]] = await prisma.$transaction([
+ prisma.$queryRaw`SELECT set_config('statement_timeout', ${String(STAGING_METRICS_STATEMENT_TIMEOUT_MS)}, true)`,
+ prisma.$queryRaw<StagingMetricsRow[]>`
+ SELECT
+ count(*) FILTER (WHERE failed_at IS NULL) AS pending,
+ count(*) FILTER (WHERE failed_at IS NOT NULL) AS failed,
+ min(created_at) FILTER (WHERE failed_at IS NULL) AS oldest_pending
+ FROM audit_log_staging
+ `,
+ ]);
+ return metrics;
+}
+
+function describeError(error: unknown): string {
+ return (error instanceof Error ? error.message : String(error)).slice(
+ 0,
+ LAST_ERROR_MAX_LENGTH,
+ );
+}
+
+export type AuditLogRecordInserter = (
+ records: AuditLogRecordInsertType[],
+ abortSignal: AbortSignal,
+) => Promise<void>;
+
+export type AuditLogRelayBatchResult = {
+ claimed: number;
+ relayed: number;
+ failed: number;
+};
+
+type ClaimedStagingRow = {
+ id: string;
+ created_at: Date;
+ payload: Prisma.JsonValue;
+};
+
+type FailedStagingRow = { id: string; error: string };
+
+type BatchOutcome = {
+ claimed: number;
+ relayed: number;
+ failures: FailedStagingRow[];
+};
+
+type StagingMetricsRow = {
+ pending: bigint;
+ failed: bigint;
+ oldest_pending: Date | null;
+};
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 149ee32c02..150510c4bf 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -21,6 +21,7 @@ import {
traceDeleteBatchActionRunner,
inAppAgentIntegrityRunner,
deletedMaskCleaner,
+ auditLogRelay,
queueMetricsRunner,
monitorRunners,
inAppAgentDlqRetryRunner,
@@ -85,6 +86,8 @@ const runDrainAndClose = async () => {
// Stop deleted-mask cleaner
deletedMaskCleaner?.stop();
+ auditLogRelay?.stop();
+
// Stop queue metrics runner
queueMetricsRunner?.stop();
traceBatchMetricsRunner?.stop();