Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
test(audit-log-stream): add e2e coverage for stream health and the delivery-failed event
Calvin2c8addf22094
diff --git a/backend/e2e-test/audit-log-stream-outbox.spec.ts b/backend/e2e-test/audit-log-stream-outbox.spec.tsnew file mode 100644index 0000000000..e112157cc9--- /dev/null+++ b/backend/e2e-test/audit-log-stream-outbox.spec.ts@@ -0,0 +1,329 @@+import { randomUUID } from "node:crypto";++import { createFakeWebhookServer, TFakeWebhookServer } from "e2e-test/fakes/webhook-destination";+import { Knex } from "knex";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";++import { TableName } from "@app/db/schemas";+import { seedData1 } from "@app/db/seed-data";+import { auditLogStreamDALFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-dal";+import { LogProvider, StreamMode } from "@app/ee/services/audit-log-stream/audit-log-stream-enums";+import {+ AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,+ AuditLogStreamDeliveryFailedPayloadSchema,+ emitAuditLogStreamDeliveryFailed+} from "@app/ee/services/audit-log-stream/audit-log-stream-events";+import { encryptLogStreamCredentials } from "@app/ee/services/audit-log-stream/audit-log-stream-fns";+import { auditLogStreamOutboxDALFactory } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-dal";+import { auditLogStreamOutboxServiceFactory } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-service";+import { AuditLogStreamOutboxStatus } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-types";+import { initEnvConfig } from "@app/lib/config/env";+import { initLogger, logger } from "@app/lib/logger";+import { eventOutboxDALFactory } from "@app/services/event-outbox/event-outbox-dal";+import { eventOutboxRegistryFactory } from "@app/services/event-outbox/event-outbox-registry";+import { eventOutboxServiceFactory } from "@app/services/event-outbox/event-outbox-service";++declare const testDb: Knex;++// Unit tests mock the DAL, so they cannot show that the stream health flip, the dropped rows and the+// alert event really commit or roll back together, that two workers racing for the same drop emit+// once, or that a receiver answering 500 ends in a failing stream. Real Postgres and a real HTTP+// receiver can. Retries are not waited out: rows are seeded at attempts = 4, so the next failure is the+// one that exhausts them, and the tests call the service and DAL directly instead of going through the+// 5s flush debounce.++const ORG_ID = seedData1.organization.id;+const CONSUMER = `e2e-als-${randomUUID().slice(0, 8)}`;+const LAST_ATTEMPT = 4;++// Stands in for the org KMS key: these tests are about the outbox, not about encryption, and the+// server does not expose its KMS service. Flip decryptShouldFail to simulate a broken key.+let decryptShouldFail = false;+const kmsService = {+ createCipherPairWithDataKey: async () => ({+ encryptor: ({ plainText }: { plainText: Buffer }) => ({ cipherTextBlob: plainText }),+ decryptor: ({ cipherTextBlob }: { cipherTextBlob: Buffer }) => {+ if (decryptShouldFail) throw new Error("kms unreachable");+ return cipherTextBlob;+ }+ })+};++const auditLogStreamDAL = auditLogStreamDALFactory(testDb as never);+const outboxDAL = auditLogStreamOutboxDALFactory(testDb as never);++const eventOutboxRegistry = eventOutboxRegistryFactory();+eventOutboxRegistry.register({+ name: CONSUMER,+ payloadSchema: AuditLogStreamDeliveryFailedPayloadSchema as never,+ subscribesTo: (eventType) => eventType === AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,+ handle: async () => []+});+const eventEmitter = eventOutboxServiceFactory({+ eventOutboxDAL: eventOutboxDALFactory(testDb as never),+ eventOutboxRegistry+});++const service = auditLogStreamOutboxServiceFactory({+ auditLogStreamOutboxDAL: outboxDAL,+ auditLogStreamDAL,+ projectDAL: { findProjectTypesByIds: async () => [] },+ kmsService: kmsService as never,+ // Required by the factory; drainStream never touches it.+ keyStore: { setItemWithExpiryNX: async () => null },+ queueService: {} as never,+ eventEmitter+});++const createdStreamIds: string[] = [];++const createStream = async (url: string) => {+ const encryptedCredentials = await encryptLogStreamCredentials({+ orgId: ORG_ID,+ credentials: { url, headers: [] },+ kmsService: kmsService as never+ });+ const [stream] = await testDb(TableName.AuditLogStream)+ .insert({ orgId: ORG_ID, provider: LogProvider.Custom, encryptedCredentials, streamMode: StreamMode.Batch })+ .returning("*");+ createdStreamIds.push(stream.id);+ return stream as { id: string };+};++const seedRow = async (+ streamId: string,+ overrides: { attempts?: number; status?: AuditLogStreamOutboxStatus; lockedAt?: Date } = {}+) => {+ const auditLogId = randomUUID();+ const [row] = await testDb(TableName.AuditLogStreamOutbox)+ .insert({+ streamId,+ orgId: ORG_ID,+ auditLogId,+ payload: JSON.stringify({ id: auditLogId, orgId: ORG_ID, eventType: "e2e.event" }),+ status: overrides.status ?? AuditLogStreamOutboxStatus.Pending,+ attempts: overrides.attempts ?? 0,+ nextRetryAt: new Date(Date.now() - 1_000),+ lockedAt: overrides.lockedAt ?? null+ })+ .returning("*");+ return Number((row as { id: string | number }).id);+};++const outboxRow = (id: number) => testDb(TableName.AuditLogStreamOutbox).where({ id }).first();+const streamHealth = (id: string) =>+ testDb(TableName.AuditLogStream).where({ id }).first("failingSince", "lastDeliveryError");+const events = () => testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).orderBy("id", "asc").select("*");++const drain = (streamId: string) => service.drainStream({ streamId, orgId: ORG_ID, provider: LogProvider.Custom });++const cleanup = async () => {+ await testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).del();+ if (createdStreamIds.length > 0) {+ await testDb(TableName.AuditLogStreamOutbox).whereIn("streamId", createdStreamIds).del();+ await testDb(TableName.AuditLogStream).whereIn("id", createdStreamIds).del();+ createdStreamIds.length = 0;+ }+};++describe("audit log stream outbox (postgres)", () => {+ beforeAll(async () => {+ // Spec files run in a separate module graph from the test server, so the logger and env config the+ // outbox service and the providers read have to be initialized here as well.+ initLogger();+ await initEnvConfig(testHsmService, testKmsRootConfigDAL, testSuperAdminDAL, logger);+ await cleanup();+ });+ afterEach(cleanup);+ afterAll(cleanup);++ describe("stream health and the alert event commit with the drop", () => {+ test("markStreamFailing flips a healthy stream exactly once under concurrent workers", async () => {+ const stream = await createStream("http://127.0.0.1:1/");++ const results = await Promise.all(+ Array.from({ length: 5 }, (_, i) =>+ outboxDAL.transaction((tx) => outboxDAL.markStreamFailing({ streamId: stream.id, errorMessage: `w${i}` }, tx))+ )+ );++ expect(results.filter((flippedAt) => flippedAt !== null)).toHaveLength(1);+ const health = await streamHealth(stream.id);+ expect(health?.failingSince).toBeInstanceOf(Date);+ expect(health?.lastDeliveryError).toMatch(/^w[0-4]$/);+ });++ test("clearStreamFailing resets both columns", async () => {+ const stream = await createStream("http://127.0.0.1:1/");+ await outboxDAL.transaction((tx) =>+ outboxDAL.markStreamFailing({ streamId: stream.id, errorMessage: "boom" }, tx)+ );++ await outboxDAL.transaction((tx) => outboxDAL.clearStreamFailing(stream.id, tx));++ expect(await streamHealth(stream.id)).toEqual({ failingSince: null, lastDeliveryError: null });+ });++ test("a rollback undoes the dropped rows, the failing flag and the event together", async () => {+ const stream = await createStream("http://127.0.0.1:1/");+ const rowId = await seedRow(stream.id, { attempts: LAST_ATTEMPT, status: AuditLogStreamOutboxStatus.Processing });++ await expect(+ outboxDAL.transaction(async (tx) => {+ await outboxDAL.commitDeliveryResult({ successIds: [], retriable: null, exhaustedIds: [rowId] }, tx);+ const failingSince = await outboxDAL.markStreamFailing({ streamId: stream.id, errorMessage: "boom" }, tx);+ await emitAuditLogStreamDeliveryFailed(+ eventEmitter,+ {+ orgId: ORG_ID,+ streamId: stream.id,+ provider: LogProvider.Custom,+ errorMessage: "boom",+ droppedCount: 1,+ failingSince: failingSince as Date+ },+ tx+ );+ throw new Error("force rollback");+ })+ ).rejects.toThrow("force rollback");++ expect((await outboxRow(rowId))?.status).toBe(AuditLogStreamOutboxStatus.Processing);+ expect(await streamHealth(stream.id)).toEqual({ failingSince: null, lastDeliveryError: null });+ expect(await events()).toHaveLength(0);+ });+ });++ describe("drainStream against a real receiver", () => {+ let receiver: TFakeWebhookServer;++ beforeAll(async () => {+ receiver = await createFakeWebhookServer();+ });+ afterAll(async () => {+ await receiver.stop();+ });+ beforeEach(() => {+ receiver.reset();+ decryptShouldFail = false;+ });++ test("delivers a healthy stream's rows and leaves its health and the event table alone", async () => {+ const stream = await createStream(receiver.url);+ const rowId = await seedRow(stream.id);++ await drain(stream.id);++ expect((await outboxRow(rowId))?.status).toBe(AuditLogStreamOutboxStatus.Delivered);+ expect(receiver.messages()).toHaveLength(1);+ expect(Array.isArray(receiver.messages()[0].body)).toBe(true);+ expect(await streamHealth(stream.id)).toEqual({ failingSince: null, lastDeliveryError: null });+ expect(await events()).toHaveLength(0);+ });++ test("a failure that can still retry does not flag the stream or emit", async () => {+ receiver.setStatusCode(500);+ const stream = await createStream(receiver.url);+ const rowId = await seedRow(stream.id, { attempts: 0 });++ await drain(stream.id);++ const row = await outboxRow(rowId);+ expect(row?.status).toBe(AuditLogStreamOutboxStatus.Retry);+ expect(row?.attempts).toBe(1);+ expect(await streamHealth(stream.id)).toEqual({ failingSince: null, lastDeliveryError: null });+ expect(await events()).toHaveLength(0);+ });++ test("exhausting a row drops it, flags the stream and emits one event", async () => {+ receiver.setStatusCode(500);+ const stream = await createStream(receiver.url);+ const rowIds = [+ await seedRow(stream.id, { attempts: LAST_ATTEMPT }),+ await seedRow(stream.id, { attempts: LAST_ATTEMPT })+ ];++ await drain(stream.id);++ expect(await Promise.all(rowIds.map(outboxRow))).toEqual([undefined, undefined]);+ const health = await streamHealth(stream.id);+ expect(health?.failingSince).toBeInstanceOf(Date);+ expect(health?.lastDeliveryError).toContain("500");++ const emitted = await events();+ expect(emitted).toHaveLength(1);+ expect(emitted[0].eventType).toBe(AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT);+ expect(emitted[0].payload).toMatchObject({+ orgId: ORG_ID,+ resourceType: "audit-log.stream",+ resourceId: stream.id,+ targetIds: [stream.id],+ provider: LogProvider.Custom,+ droppedCount: 2+ });+ expect(AuditLogStreamDeliveryFailedPayloadSchema.safeParse(emitted[0].payload).success).toBe(true);+ });++ test("a stream that stays down does not emit again, and keeps its first failure time", async () => {+ receiver.setStatusCode(500);+ const stream = await createStream(receiver.url);+ await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ await drain(stream.id);+ const first = await streamHealth(stream.id);++ await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ await drain(stream.id);++ expect(await events()).toHaveLength(1);+ expect((await streamHealth(stream.id))?.failingSince).toEqual(first?.failingSince);+ });++ test("a delivery after the outage clears the failing state without emitting", async () => {+ receiver.setStatusCode(500);+ const stream = await createStream(receiver.url);+ await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ await drain(stream.id);+ expect((await streamHealth(stream.id))?.failingSince).toBeInstanceOf(Date);++ receiver.setStatusCode(200);+ const rowId = await seedRow(stream.id);+ await drain(stream.id);++ expect((await outboxRow(rowId))?.status).toBe(AuditLogStreamOutboxStatus.Delivered);+ expect(await streamHealth(stream.id)).toEqual({ failingSince: null, lastDeliveryError: null });+ expect(await events()).toHaveLength(1);+ });++ test("a stream that fails again after recovering emits a new event", async () => {+ receiver.setStatusCode(500);+ const stream = await createStream(receiver.url);+ await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ await drain(stream.id);++ receiver.setStatusCode(200);+ await seedRow(stream.id);+ await drain(stream.id);++ receiver.setStatusCode(500);+ await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ await drain(stream.id);++ expect(await events()).toHaveLength(2);+ });++ test("credentials that cannot be decrypted end in a failing stream instead of stuck rows", async () => {+ const stream = await createStream(receiver.url);+ const rowId = await seedRow(stream.id, { attempts: LAST_ATTEMPT });+ decryptShouldFail = true;++ await drain(stream.id);++ expect(receiver.messages()).toHaveLength(0);+ expect(await outboxRow(rowId)).toBeUndefined();+ const health = await streamHealth(stream.id);+ expect(health?.failingSince).toBeInstanceOf(Date);+ expect(health?.lastDeliveryError).toContain("Failed to decrypt stream credentials");+ expect(await events()).toHaveLength(1);+ });+ });+});diff --git a/backend/e2e-test/fakes/webhook-destination.ts b/backend/e2e-test/fakes/webhook-destination.tsindex 4049c3617f..05303dd906 100644--- a/backend/e2e-test/fakes/webhook-destination.ts+++ b/backend/e2e-test/fakes/webhook-destination.ts@@ -20,6 +20,8 @@ export type TFakeWebhookServer = {// messages then arrive interleaved and out of any one test's control. Rejects if none arrives// within timeoutMs.waitForMessage: (predicate?: TMessagePredicate, timeoutMs?: number) => Promise<TFakeWebhookMessage>;+ // Status the fake answers with from now on (default 200), for exercising a destination that fails.+ setStatusCode: (statusCode: number) => void;reset: () => void;stop: () => Promise<void>;};@@ -33,6 +35,7 @@ const matchAny: TMessagePredicate = () => true;export const createFakeWebhookServer = async (): Promise<TFakeWebhookServer> => {const messages: TFakeWebhookMessage[] = [];const claimed = new Set<TFakeWebhookMessage>();+ let statusCode = 200;const waiters: { predicate: TMessagePredicate; resolve: (message: TFakeWebhookMessage) => void }[] = [];const server = http.createServer((req, res) => {@@ -65,8 +68,8 @@ export const createFakeWebhookServer = async (): Promise<TFakeWebhookServer> =>waiter.resolve(message);}- res.writeHead(200, { "Content-Type": "application/json" });- res.end(JSON.stringify({ ok: true }));+ res.writeHead(statusCode, { "Content-Type": "application/json" });+ res.end(JSON.stringify({ ok: statusCode < 400 }));});});@@ -107,9 +110,13 @@ export const createFakeWebhookServer = async (): Promise<TFakeWebhookServer> =>url: `http://127.0.0.1:${port}/`,messages: () => messages,waitForMessage,+ setStatusCode: (code: number) => {+ statusCode = code;+ },reset: () => {messages.length = 0;claimed.clear();+ statusCode = 200;},stop: () =>new Promise<void>((resolve, reject) => {