Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
import type { ClickHouseClient } from "@clickhouse/client";
import { randomUUID } from "crypto";
import { v7 as uuidv7 } from "uuid";
import type { TAuditLogs } from "@app/db/schemas";
import { TAuditLogStreamOutboxServiceFactory } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { requestMemoKeys } from "@app/lib/request-context/memo-keys";
import { requestMemoize } from "@app/lib/request-context/request-memoizer";
import {
auditLogDroppedCounter,
auditLogEnqueuedCounter,
auditLogPersistDurationHistogram
} from "@app/lib/telemetry/metrics";
import { sanitizeLogPayload, sanitizeLogText } from "@app/lib/validator/log-safe-text";
import { JOB_SCHEDULER_PREFIX, QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { TAuditLogDALFactory } from "./audit-log-dal";
import { getAuditLogEventClass, isAlwaysRecordedEventClass } from "./audit-log-event-classes";
import { getAuditLogExpiresAt, hasAuditLogRetention } from "./audit-log-retention-fns";
import { isAuditLogEventEnabled, TAuditLogSettingsServiceFactory } from "./audit-log-settings-service";
import { TAuditLogStreamEntry, TCreateAuditLogDTO } from "./audit-log-types";
type TAuditLogQueueServiceFactoryDep = {
auditLogDAL: TAuditLogDALFactory;
auditLogStreamOutboxService: Pick<TAuditLogStreamOutboxServiceFactory, "enqueueForLogs">;
queueService: TQueueServiceFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
auditLogSettingsService: Pick<TAuditLogSettingsServiceFactory, "getEffectiveSettings">;
clickhouseClient: ClickHouseClient | null;
keyStore: Pick<TKeyStoreFactory, "streamAdd" | "streamCollect" | "streamTrim" | "acquireLock">;
};
export type TAuditLogQueueServiceFactory = {
pushToLog: (data: TCreateAuditLogDTO) => Promise<void>;
pushToLogOrThrow: (data: TCreateAuditLogDTO) => Promise<void>;
};
const normalizeJsonPayload = (payload: unknown) => {
if (Array.isArray(payload)) {
return { data: payload };
}
if (typeof payload === "object" && payload !== null) {
return payload;
}
logger.error({ payload }, "audit-log-queue: Unexpected audit log payload type, expected object or array");
return {};
};
const normalizeEmptyValue = (value: string | undefined | null, isClickhouse: boolean) => {
if (!value) {
return isClickhouse ? "" : null;
}
return value;
};
// keep this timeout 5s it must be fast because else the queue will take time to finish
// audit log is a crowded queue thus needs to be fast
export const AUDIT_LOG_STREAM_TIMEOUT = 5 * 1000;
// batch deliveries can carry up to AUDIT_LOG_STREAM_BATCH_SIZE events to remote intakes;
// give them more headroom than the single-event ping timeout above.
export const AUDIT_LOG_STREAM_BATCH_TIMEOUT = 30 * 1000;
const AUDIT_LOG_STREAM_KEY = "audit-log-stream";
const AUDIT_LOG_STREAM_BATCH_SIZE = 10_000;
const AUDIT_LOG_STREAM_MAX_ENTRIES = 50_000;
const AUDIT_LOG_STREAM_CONSUMER_LOCK_TTL_MS = 60_000;
export const auditLogQueueServiceFactory = async ({
auditLogDAL,
queueService,
projectDAL,
licenseService,
auditLogSettingsService,
auditLogStreamOutboxService,
clickhouseClient,
keyStore
}: TAuditLogQueueServiceFactoryDep): Promise<TAuditLogQueueServiceFactory> => {
const { CLICKHOUSE_AUDIT_LOG_ENABLED, CLICKHOUSE_AUDIT_LOG_TABLE_NAME, CLICKHOUSE_AUDIT_LOG_INSERT_SETTINGS } =
getConfig();
const isClickHouseBatchEnabled = Boolean(clickhouseClient && CLICKHOUSE_AUDIT_LOG_ENABLED);
const buildStreamEntry = async (data: TCreateAuditLogDTO): Promise<TAuditLogStreamEntry | null> => {
const { projectId } = data;
if (!data.orgId && !projectId) {
logger.error(`audit-log-queue: Skipping entry with neither orgId nor projectId [event=${data.event?.type}]`);
return null;
}
const project =
!data.orgId && projectId
? await requestMemoize(requestMemoKeys.projectFindById(projectId), () => projectDAL.findById(projectId))
: undefined;
if (!data.orgId && projectId && !project) {
logger.error(
`audit-log-queue: Skipping entry, project was deleted [projectId=${projectId}] [event=${data.event?.type}]`
);
return null;
}
const orgId = data.orgId ?? project?.orgId;
if (!orgId) return null;
if (!isAlwaysRecordedEventClass(getAuditLogEventClass(data.event.type))) {
const settings = await requestMemoize(requestMemoKeys.auditLogSettings(orgId, projectId), () =>
auditLogSettingsService.getEffectiveSettings(orgId, projectId)
);
if (!isAuditLogEventEnabled(settings, data.event.type, projectId)) return null;
}
const plan = await requestMemoize(requestMemoKeys.licensePlan(orgId), () => licenseService.getPlan(orgId));
if (!plan || !hasAuditLogRetention(plan)) return null;
const createdAt = new Date();
const id = isClickHouseBatchEnabled ? uuidv7({ msecs: createdAt.getTime() }) : randomUUID();
return {
...data,
id,
orgId,
createdAt: createdAt.toISOString(),
expiresAt: getAuditLogExpiresAt(createdAt, plan, project?.auditLogsRetentionDays).toISOString(),
projectName: project?.name
};
};
// Append the resolved log to the Redis ingest stream. A dropped (null) entry is a no-op.
const appendToIngestStream = async (data: TCreateAuditLogDTO) => {
const entry = await buildStreamEntry(data);
if (!entry) return;
await keyStore.streamAdd(AUDIT_LOG_STREAM_KEY, "*", { data: JSON.stringify(entry) });
auditLogEnqueuedCounter.add(1, {
"audit_log.event_type": entry.event?.type ?? "unknown",
"audit_log.actor_type": entry.actor?.type ?? "unknown"
});
};
// Request-path push: a transient Redis/DB failure must not fail the request that produced
// the audit event, so errors (resolution lookups included) are logged and swallowed
// (at-most-once on this path).
const pushToLog = async (data: TCreateAuditLogDTO) => {
try {
await appendToIngestStream(data);
} catch (error) {
// At-most-once on the request path: the produced event is now lost (not retried). Log it
// as an explicit drop so it's searchable as audit-log data loss, not a generic push failure.
logger.error(
error,
`audit-log-queue: Dropped audit log — failed to push to ingest stream (at-most-once on request path) [event=${data.event?.type}] [orgId=${data.orgId}] [projectId=${data.projectId}]`
);
auditLogDroppedCounter.add(1, {
"audit_log.event_type": data.event?.type ?? "unknown",
"audit_log.actor_type": data.actor?.type ?? "unknown",
"audit_log.drop_reason": "ingest_stream_push_failed",
...(data.orgId ? { "infisical.organization.id": data.orgId } : {})
});
}
};
const insertBatch = async (logs: TAuditLogs[]) => {
if (isClickHouseBatchEnabled) {
await clickhouseClient!.insert({
table: CLICKHOUSE_AUDIT_LOG_TABLE_NAME,
clickhouse_settings: CLICKHOUSE_AUDIT_LOG_INSERT_SETTINGS,
values: logs,
format: "JSONEachRow"
});
} else {
await auditLogDAL.batchCreate(logs);
}
};
// Unified consumer: drain a batch from the ingest stream, map each (already-resolved) entry to
// an audit-log row, batch-insert into the active backend, fan out to the stream outbox, and
// trim. No DB lookups happen here — org/plan/retention were resolved at push time. Trim happens only after a
// successful insert so a failed batch is retried on the next tick (re-inserts are idempotent:
// ClickHouse ReplacingMergeTree dedups on id, Postgres uses ON CONFLICT (id) DO NOTHING).
const drainAuditLogStream = async () => {
const { entries, lastId } = await keyStore.streamCollect(
AUDIT_LOG_STREAM_KEY,
AUDIT_LOG_STREAM_BATCH_SIZE,
AUDIT_LOG_STREAM_MAX_ENTRIES
);
if (entries.length === 0 || !lastId) return;
if (entries.length >= AUDIT_LOG_STREAM_MAX_ENTRIES) {
logger.warn(
`audit-log-queue: ingest stream backlog hit the collection ceiling [collected=${entries.length}] [maxEntries=${AUDIT_LOG_STREAM_MAX_ENTRIES}] — stream is backing up; oldest entries risk being dropped by the Redis MAXLEN cap`
);
}
// Parse — a single corrupt entry is skipped, not allowed to abort the whole batch.
const parsed: unknown[] = [];
for (const [, fields] of entries) {
try {
parsed.push(JSON.parse(fields[1]));
} catch (error) {
logger.error(error, "audit-log-queue: Skipping unparseable audit log stream entry");
}
}
// Map a stream entry into a persistable audit log. org/plan/TTL/projectName were all
// resolved at push time (see buildStreamEntry), so this is a pure shape transform with no DB
// lookups. Entries that predate push-time resolution (in-flight during a rolling deploy) lack
// `orgId`/`expiresAt`; drop them rather than persist a row with a missing org or expiry.
const enrichEntry = (raw: unknown): TAuditLogs | null => {
if (typeof raw !== "object" || raw === null) {
logger.error("audit-log-queue: Skipping non-object audit log stream entry");
return null;
}
const entry = raw as TAuditLogStreamEntry;
if (!entry.orgId || !entry.expiresAt) {
logger.error(
`audit-log-queue: Skipping entry missing resolved metadata [event=${entry.event?.type}] [orgId=${entry.orgId}]`
);
return null;
}
const createdAt = new Date(entry.createdAt);
// `enriched` feeds both the insert and the stream outbox, so cleaning here covers both.
return {
id: entry.id,
actor: entry.actor.type,
actorMetadata: sanitizeLogPayload(normalizeJsonPayload(entry.actor.metadata)),
ipAddress: normalizeEmptyValue(sanitizeLogText(entry.ipAddress), isClickHouseBatchEnabled),
eventType: entry.event.type,
eventMetadata: sanitizeLogPayload(normalizeJsonPayload(entry.event.metadata)),
userAgent: normalizeEmptyValue(sanitizeLogText(entry.userAgent), isClickHouseBatchEnabled),
userAgentType: normalizeEmptyValue(entry.userAgentType, isClickHouseBatchEnabled),
projectId: normalizeEmptyValue(entry.projectId, isClickHouseBatchEnabled),
orgId: entry.orgId,
expiresAt: new Date(entry.expiresAt),
createdAt,
updatedAt: createdAt,
// project name is only stored for non-ClickHouse insertion
...(!isClickHouseBatchEnabled ? { projectName: sanitizeLogText(entry.projectName) } : {})
};
};
// Per-entry guard: a single unmappable entry (corrupt or unexpected shape) is skipped,
// never allowed to throw out of the map and abort the whole batch
const enriched: TAuditLogs[] = [];
for (const raw of parsed) {
try {
const log = enrichEntry(raw);
if (log) enriched.push(log);
} catch (error) {
logger.error(error, "audit-log-queue: Skipping unmappable audit log stream entry");
}
}
const backend = isClickHouseBatchEnabled ? "clickhouse" : "postgres";
if (enriched.length > 0) {
const persistStart = Date.now();
try {
await insertBatch(enriched);
} catch (error) {
auditLogPersistDurationHistogram.record((Date.now() - persistStart) / 1000, {
"audit_log.backend": backend,
outcome: "failure"
});
logger.error(
error,
`audit-log-queue: Failed to batch insert ${enriched.length} audit logs [backend=${backend}]`
);
// Do not trim — the same entries are reprocessed next tick (re-insert is idempotent).
return;
}
auditLogPersistDurationHistogram.record((Date.now() - persistStart) / 1000, {
"audit_log.backend": backend,
outcome: "success"
});
if (getConfig().AUDIT_LOG_STREAMS_ENABLED) {
try {
await auditLogStreamOutboxService.enqueueForLogs(enriched);
} catch (error) {
logger.error(
error,
"audit-log-queue: Failed to enqueue audit logs to stream outbox; skipping trim so the batch is reprocessed"
);
return;
}
}
logger.info(
{
inserted: enriched.length,
dropped: parsed.length - enriched.length,
backend
},
"audit-log-queue: Batch processed audit logs"
);
}
// Trim whenever the insert step did not throw — including the all-dropped case — so skipped
// and handled entries don't accumulate in the stream forever.
await keyStore.streamTrim(AUDIT_LOG_STREAM_KEY, lastId, true);
};
// Cron fires every 5 s but a tick can run longer under load (big batch, slow DB). Multiple
// pods all subscribe to the scheduler so two ticks can overlap. The work is idempotent, but
// overlap doubles DB / Redis pressure when we're already behind. Guard the body with redlock
// (`retryCount: 0` makes it non-blocking): if another runner holds the lock we just skip —
// the next scheduled tick will pick up where it left off. The TTL is a safety net for a
// crashed consumer; release uses a fencing-token Lua script so a slow runner whose lock
// already expired can't accidentally release a successor's lock.
const consumeAuditLogStream = async () => {
let lock;
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.AuditLogIngestConsumerLock],
AUDIT_LOG_STREAM_CONSUMER_LOCK_TTL_MS,
{
retryCount: 0
}
);
} catch {
// Another runner holds the lock — skip this tick.
return;
}
try {
await drainAuditLogStream();
} finally {
await lock.release();
}
};
// Single scheduled consumer for both backends, every 5 seconds. Registered unconditionally
// now that Postgres also batches through the stream. The scheduler id is kept as the historical
// "audit-log-clickhouse-batch" so the existing Redis scheduler key isn't orphaned on upgrade.
queueService.start(QueueName.AuditLogClickHouseBatch, consumeAuditLogStream);
// Only the fleet that consumes this queue arms its scheduler. An api or secret-scanning pod has
// no worker on it, so registering from there would re-arm a 5s schedule nobody drains.
if (getConfig().isGeneralWorkerRunModeEnabled) {
await queueService.upsertJobScheduler(
QueueName.AuditLogClickHouseBatch,
`${JOB_SCHEDULER_PREFIX}:audit-log-clickhouse-batch`,
{ every: 5000 },
{ name: QueueJobs.AuditLogClickHouseBatch }
);
}
return {
pushToLog,
pushToLogOrThrow: appendToIngestStream
};
};