Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
test(audit-log-stream): add an end-to-end failure alert test and shorten outbox timings in e2e
Calvindef542c871f7
diff --git a/backend/e2e-test/audit-log-stream-outbox.spec.ts b/backend/e2e-test/audit-log-stream-outbox.integration.spec.tssimilarity index 91%rename from backend/e2e-test/audit-log-stream-outbox.spec.tsrename to backend/e2e-test/audit-log-stream-outbox.integration.spec.tsindex a9b1dd73af..033f20a471 100644--- a/backend/e2e-test/audit-log-stream-outbox.spec.ts+++ b/backend/e2e-test/audit-log-stream-outbox.integration.spec.ts@@ -1,11 +1,11 @@import { randomUUID } from "node:crypto";import { createFakeWebhookServer, TFakeWebhookServer } from "e2e-test/fakes/webhook-destination";+import { createIsolatedOrgAndProject } from "e2e-test/testUtils/fixtures";import { Knex } from "knex";import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";import { TableName } from "@app/db/schemas";-import { seedData1 } from "@app/db/seed-data";import { auditLogStreamDALFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-dal";import { LogProvider, StreamMode } from "@app/ee/services/audit-log-stream/audit-log-stream-enums";import {@@ -25,6 +25,12 @@ import { eventOutboxServiceFactory } from "@app/services/event-outbox/event-outbdeclare const testDb: Knex;+// This is an integration test, not an end-to-end one: it builds the outbox service on the test+// database and calls it directly, so it never goes through the HTTP API. Streams are created with+// the test DB because the e2e license has audit log streams off and nothing over HTTP triggers a+// drain or exposes a stream's failing state. The org is a fresh isolated one, deleted at the end,+// which cascades to its streams and outbox rows.+//// Unit tests mock the DAL, so they cannot show that the stream health flip, the dropped rows and the// alert event really commit or roll back together, that two workers racing for the same drop emit// once, or that a receiver answering 500 ends in a failing stream. Real Postgres and a real HTTP@@ -32,7 +38,8 @@ declare const testDb: Knex;// one that exhausts them, and the tests call the service and DAL directly instead of going through the// 5s flush debounce.-const ORG_ID = seedData1.organization.id;+let orgId: string;+let cleanupOrg: () => Promise<void>;const CONSUMER = `e2e-als-${randomUUID().slice(0, 8)}`;const LAST_ATTEMPT = 4;@@ -75,18 +82,15 @@ const service = auditLogStreamOutboxServiceFactory({eventEmitter});-const createdStreamIds: string[] = [];-const createStream = async (url: string) => {const encryptedCredentials = await encryptLogStreamCredentials({- orgId: ORG_ID,+ orgId,credentials: { url, headers: [] },kmsService: kmsService as never});const [stream] = await testDb(TableName.AuditLogStream)- .insert({ orgId: ORG_ID, provider: LogProvider.Custom, encryptedCredentials, streamMode: StreamMode.Batch })+ .insert({ orgId, provider: LogProvider.Custom, encryptedCredentials, streamMode: StreamMode.Batch }).returning("*");- createdStreamIds.push(stream.id);return stream as { id: string };};@@ -98,9 +102,9 @@ const seedRow = async (const [row] = await testDb(TableName.AuditLogStreamOutbox).insert({streamId,- orgId: ORG_ID,+ orgId,auditLogId,- payload: JSON.stringify({ id: auditLogId, orgId: ORG_ID, eventType: "e2e.event" }),+ payload: JSON.stringify({ id: auditLogId, orgId, eventType: "e2e.event" }),status: overrides.status ?? AuditLogStreamOutboxStatus.Pending,attempts: overrides.attempts ?? 0,nextRetryAt: new Date(Date.now() - 1_000),@@ -115,16 +119,9 @@ const streamHealth = (id: string) =>testDb(TableName.AuditLogStream).where({ id }).first("failingSince", "lastDeliveryError");const events = () => testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).orderBy("id", "asc").select("*");-const drain = (streamId: string) => service.drainStream({ streamId, orgId: ORG_ID, provider: LogProvider.Custom });+const drain = (streamId: string) => service.drainStream({ streamId, orgId, provider: LogProvider.Custom });-const cleanup = async () => {- await testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).del();- if (createdStreamIds.length > 0) {- await testDb(TableName.AuditLogStreamOutbox).whereIn("streamId", createdStreamIds).del();- await testDb(TableName.AuditLogStream).whereIn("id", createdStreamIds).del();- createdStreamIds.length = 0;- }-};+const cleanupEvents = () => testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).del();describe("audit log stream outbox (postgres)", () => {beforeAll(async () => {@@ -132,10 +129,14 @@ describe("audit log stream outbox (postgres)", () => {// outbox service and the providers read have to be initialized here as well.initLogger();await initEnvConfig(testHsmService, testKmsRootConfigDAL, testSuperAdminDAL, logger);- await cleanup();+ ({ orgId, cleanup: cleanupOrg } = await createIsolatedOrgAndProject("audit-log-stream-outbox-e2e"));+ await cleanupEvents();+ });+ afterEach(cleanupEvents);+ afterAll(async () => {+ await cleanupEvents();+ await cleanupOrg();});- afterEach(cleanup);- afterAll(cleanup);describe("stream health and the alert event commit with the drop", () => {test("markStreamFailing flips a healthy stream exactly once under concurrent workers", async () => {@@ -175,7 +176,7 @@ describe("audit log stream outbox (postgres)", () => {await emitAuditLogStreamDeliveryFailed(eventEmitter,{- orgId: ORG_ID,+ orgId,streamId: stream.id,provider: LogProvider.Custom,errorMessage: "boom",@@ -254,7 +255,7 @@ describe("audit log stream outbox (postgres)", () => {expect(emitted).toHaveLength(1);expect(emitted[0].eventType).toBe(AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT);expect(emitted[0].payload).toMatchObject({- orgId: ORG_ID,+ orgId,resourceType: "audit-log.stream",resourceId: null,targetIds: [stream.id],diff --git a/backend/e2e-test/fakes/audit-log-stream-outbox-constants.ts b/backend/e2e-test/fakes/audit-log-stream-outbox-constants.tsnew file mode 100644index 0000000000..d0918d91a7--- /dev/null+++ b/backend/e2e-test/fakes/audit-log-stream-outbox-constants.ts@@ -0,0 +1,44 @@+import * as RealConstants from "../../src/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-constants";++// Replaces the outbox's debounce and retry limit so a spec can watch a stream fail without waiting+// out the real timings (a 5s debounce, then five attempts spread over backoff). Wired up by+// test.alias in vitest.e2e.config.mts; nothing under src/ references this file.+//+// Defaults are the real values, so specs that do not ask for anything else see production+// behavior. The values are live bindings: the service reads them at call time, so a change made+// through the handle below applies to the next flush.+//+// The handle hangs off globalThis for the reason given in aws-parameter-store-sync-fns.ts.+/* eslint-disable import/no-mutable-exports -- a live binding is what lets the service see a change */+export let FLUSH_DEBOUNCE_MS = RealConstants.FLUSH_DEBOUNCE_MS as number;+export let MAX_ATTEMPTS = RealConstants.MAX_ATTEMPTS as number;+/* eslint-enable import/no-mutable-exports */++const globalScope = globalThis as typeof globalThis & {+ infisicalAuditLogStreamOutboxTuning?: {+ set: (opts: { flushDebounceMs?: number; maxAttempts?: number }) => void;+ reset: () => void;+ };+};++const reset = () => {+ FLUSH_DEBOUNCE_MS = RealConstants.FLUSH_DEBOUNCE_MS;+ MAX_ATTEMPTS = RealConstants.MAX_ATTEMPTS;+};++globalScope.infisicalAuditLogStreamOutboxTuning ??= {+ set: (opts) => {+ FLUSH_DEBOUNCE_MS = opts.flushDebounceMs ?? FLUSH_DEBOUNCE_MS;+ MAX_ATTEMPTS = opts.maxAttempts ?? MAX_ATTEMPTS;+ },+ reset+};++export const auditLogStreamOutboxTuning = {+ set: (opts: { flushDebounceMs?: number; maxAttempts?: number }) => {+ globalScope.infisicalAuditLogStreamOutboxTuning?.set(opts);+ },+ reset: () => {+ globalScope.infisicalAuditLogStreamOutboxTuning?.reset();+ }+};diff --git a/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.ts b/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.tsnew file mode 100644index 0000000000..60de388925--- /dev/null+++ b/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.ts@@ -0,0 +1,113 @@+import { auditLogStreamOutboxTuning } from "e2e-test/fakes/audit-log-stream-outbox-constants";+import { eventOutbox } from "e2e-test/fakes/event-outbox-queue";+import { createFakeWebhookServer, TFakeWebhookServer } from "e2e-test/fakes/webhook-destination";+import { TTestSmtpService } from "e2e-test/mocks/smtp";+import { createAlert } from "e2e-test/testUtils/alerts";+import { createIsolatedOrgAndProject } from "e2e-test/testUtils/fixtures";+import { overrideLicenseFeatures } from "e2e-test/testUtils/license";+import { pollUntil } from "e2e-test/testUtils/poll";+import { createSecretV2 } from "e2e-test/testUtils/secrets";+import { addUserMembership, createUser, deleteUsers } from "e2e-test/testUtils/users";++import { OrgMembershipRole } from "@app/db/schemas";+import {+ AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,+ AUDIT_LOG_STREAM_RESOURCE_TYPE+} from "@app/ee/services/audit-log-stream/audit-log-stream-events";+import { ALERT_EVENT_CONSUMER } from "@app/services/alert/alert-event-consumer";++const smtp = () => (globalThis as unknown as { testSmtp: TTestSmtpService }).testSmtp;++// The one end-to-end check that a stream which cannot deliver ends in a notification. It goes+// through the public API from stream creation to the alert email; the edge cases (rollback,+// repeat failures, recovery) are covered in audit-log-stream-outbox.integration.spec.ts.+//+// The debounce and retry limit are shortened by a fake (see fakes/audit-log-stream-outbox-constants.ts)+// so the first failed delivery exhausts the row, instead of waiting out five backed-off attempts.+describe("Audit log stream failure alert", () => {+ let orgId: string;+ let projectId: string;+ let authToken: string;+ let cleanup: () => Promise<void>;+ let restoreLicense: () => void;+ let streamReceiver: TFakeWebhookServer;+ let recipient: { userId: string; username: string };++ beforeAll(async () => {+ restoreLicense = overrideLicenseFeatures({+ auditLogs: true,+ auditLogsRetentionDays: 30,+ auditLogStreams: true+ });+ auditLogStreamOutboxTuning.set({ flushDebounceMs: 1_000, maxAttempts: 1 });+ streamReceiver = await createFakeWebhookServer();+ ({ orgId, projectId, authToken, cleanup } = await createIsolatedOrgAndProject("audit-log-stream-failure-alert"));+ recipient = await createUser("stream-alert-recipient");+ await addUserMembership({ userId: recipient.userId, orgId, role: OrgMembershipRole.Member });+ });++ afterAll(async () => {+ await cleanup();+ await deleteUsers([recipient.userId]);+ await streamReceiver.stop();+ auditLogStreamOutboxTuning.reset();+ restoreLicense();+ });++ test("a stream whose destination keeps failing alerts the org", async () => {+ const streamRes = await testServer.inject({+ method: "POST",+ url: "/api/v1/audit-log-streams/custom",+ headers: { authorization: `Bearer ${authToken}` },+ body: { credentials: { url: streamReceiver.url, headers: [] } }+ });+ expect(streamRes.statusCode, streamRes.payload).toBe(200);+ const streamId = streamRes.json().auditLogStream.id as string;++ // Creating a stream sends a test request, so the destination only starts failing afterwards.+ streamReceiver.reset();+ streamReceiver.setStatusCode(500);++ await createAlert({+ authToken,+ body: {+ name: "stream failure",+ resourceType: AUDIT_LOG_STREAM_RESOURCE_TYPE,+ eventType: AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,+ channels: [+ {+ name: "Email",+ channelType: "email",+ config: {},+ recipients: [{ principalType: "user", principalId: recipient.userId }]+ }+ ]+ }+ });++ await createSecretV2({+ workspaceId: projectId,+ environmentSlug: "dev",+ secretPath: "/",+ key: "TRIGGER_AUDIT_LOG",+ value: "value",+ authToken+ });++ // The stream's first flush fails and drops the row. The alert event it emits is then delivered+ // by draining the event outbox here, rather than waiting for the relay's next tick.+ const [email] = await pollUntil({+ describe: "the delivery-failed alert email",+ read: async () => {+ await eventOutbox.drain(ALERT_EVENT_CONSUMER);+ return smtp()+ .getEmails()+ .filter((mail) => JSON.stringify(mail.substitutions).includes(streamId));+ },+ done: (emails) => emails.length > 0,+ timeoutMs: 30_000+ });++ expect(email.recipients).toEqual([recipient.username]);+ }, 40_000);+});diff --git a/backend/e2e-test/setup/reset-fakes.ts b/backend/e2e-test/setup/reset-fakes.tsindex d05c5951f2..fd7f32e7de 100644--- a/backend/e2e-test/setup/reset-fakes.ts+++ b/backend/e2e-test/setup/reset-fakes.ts@@ -1,3 +1,4 @@+import { auditLogStreamOutboxTuning } from "e2e-test/fakes/audit-log-stream-outbox-constants";import { fakeAwsConnection } from "e2e-test/fakes/aws-connection-fns";import { fakeIamUserSecret } from "e2e-test/fakes/aws-iam-user-secret-rotation-fns";import { fakeParameterStore } from "e2e-test/fakes/aws-parameter-store-sync-fns";@@ -18,4 +19,5 @@ beforeAll(() => {fakeAwsConnection.reset();fakeS3Bucket.reset();fakeIamUserSecret.reset();+ auditLogStreamOutboxTuning.reset();});diff --git a/backend/e2e-test/testUtils/license.ts b/backend/e2e-test/testUtils/license.tsnew file mode 100644index 0000000000..2bba54ff74--- /dev/null+++ b/backend/e2e-test/testUtils/license.ts@@ -0,0 +1,17 @@+import type { TFeatureSet } from "@app/ee/services/license/license-types";++// Turns plan features on for the whole server until the returned function is called, for a spec+// that exercises a paid feature. The e2e license is a fixed default (see the ./license-fns alias in+// vitest.e2e.config.mts) and getPlan hands out this one object, so the override is not scoped to an+// org: call the restore in afterAll, and keep specs that read it in the same file.+export const overrideLicenseFeatures = (features: Partial<TFeatureSet>) => {+ const { license } = (globalThis as unknown as { testServices: { license: { onPremFeatures: TFeatureSet } } })+ .testServices;+ const plan = license.onPremFeatures as unknown as Record<string, unknown>;+ const original = Object.fromEntries(Object.keys(features).map((key) => [key, plan[key]]));+ Object.assign(plan, features);++ return () => {+ Object.assign(plan, original);+ };+};diff --git a/backend/vitest.e2e.config.mts b/backend/vitest.e2e.config.mtsindex 957dd0e859..b8cb8ffa3f 100644--- a/backend/vitest.e2e.config.mts+++ b/backend/vitest.e2e.config.mts@@ -98,6 +98,11 @@ export default defineConfig({{find: /^@app\/services\/event-outbox\/event-outbox-queue$/,replacement: path.resolve(__dirname, "./e2e-test/fakes/event-outbox-queue")+ },+ // Only audit-log-stream-outbox-service.ts imports this specifier.+ {+ find: "./audit-log-stream-outbox-constants",+ replacement: path.resolve(__dirname, "./e2e-test/fakes/audit-log-stream-outbox-constants")}]},