Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

test(audit-log-stream): add an end-to-end failure alert test and shorten outbox timings in e2e

Calvindef542c871f7
diff --git a/backend/e2e-test/audit-log-stream-outbox.spec.ts b/backend/e2e-test/audit-log-stream-outbox.integration.spec.ts
similarity index 91%
rename from backend/e2e-test/audit-log-stream-outbox.spec.ts
rename to backend/e2e-test/audit-log-stream-outbox.integration.spec.ts
index a9b1dd73af..033f20a471 100644
--- a/backend/e2e-test/audit-log-stream-outbox.spec.ts
+++ b/backend/e2e-test/audit-log-stream-outbox.integration.spec.ts
@@ -1,11 +1,11 @@
import { randomUUID } from "node:crypto";
import { createFakeWebhookServer, TFakeWebhookServer } from "e2e-test/fakes/webhook-destination";
+import { createIsolatedOrgAndProject } from "e2e-test/testUtils/fixtures";
import { Knex } from "knex";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { TableName } from "@app/db/schemas";
-import { seedData1 } from "@app/db/seed-data";
import { auditLogStreamDALFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-dal";
import { LogProvider, StreamMode } from "@app/ee/services/audit-log-stream/audit-log-stream-enums";
import {
@@ -25,6 +25,12 @@ import { eventOutboxServiceFactory } from "@app/services/event-outbox/event-outb
declare const testDb: Knex;
+// This is an integration test, not an end-to-end one: it builds the outbox service on the test
+// database and calls it directly, so it never goes through the HTTP API. Streams are created with
+// the test DB because the e2e license has audit log streams off and nothing over HTTP triggers a
+// drain or exposes a stream's failing state. The org is a fresh isolated one, deleted at the end,
+// which cascades to its streams and outbox rows.
+//
// Unit tests mock the DAL, so they cannot show that the stream health flip, the dropped rows and the
// alert event really commit or roll back together, that two workers racing for the same drop emit
// once, or that a receiver answering 500 ends in a failing stream. Real Postgres and a real HTTP
@@ -32,7 +38,8 @@ declare const testDb: Knex;
// one that exhausts them, and the tests call the service and DAL directly instead of going through the
// 5s flush debounce.
-const ORG_ID = seedData1.organization.id;
+let orgId: string;
+let cleanupOrg: () => Promise<void>;
const CONSUMER = `e2e-als-${randomUUID().slice(0, 8)}`;
const LAST_ATTEMPT = 4;
@@ -75,18 +82,15 @@ const service = auditLogStreamOutboxServiceFactory({
eventEmitter
});
-const createdStreamIds: string[] = [];
-
const createStream = async (url: string) => {
const encryptedCredentials = await encryptLogStreamCredentials({
- orgId: ORG_ID,
+ orgId,
credentials: { url, headers: [] },
kmsService: kmsService as never
});
const [stream] = await testDb(TableName.AuditLogStream)
- .insert({ orgId: ORG_ID, provider: LogProvider.Custom, encryptedCredentials, streamMode: StreamMode.Batch })
+ .insert({ orgId, provider: LogProvider.Custom, encryptedCredentials, streamMode: StreamMode.Batch })
.returning("*");
- createdStreamIds.push(stream.id);
return stream as { id: string };
};
@@ -98,9 +102,9 @@ const seedRow = async (
const [row] = await testDb(TableName.AuditLogStreamOutbox)
.insert({
streamId,
- orgId: ORG_ID,
+ orgId,
auditLogId,
- payload: JSON.stringify({ id: auditLogId, orgId: ORG_ID, eventType: "e2e.event" }),
+ payload: JSON.stringify({ id: auditLogId, orgId, eventType: "e2e.event" }),
status: overrides.status ?? AuditLogStreamOutboxStatus.Pending,
attempts: overrides.attempts ?? 0,
nextRetryAt: new Date(Date.now() - 1_000),
@@ -115,16 +119,9 @@ const streamHealth = (id: string) =>
testDb(TableName.AuditLogStream).where({ id }).first("failingSince", "lastDeliveryError");
const events = () => testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).orderBy("id", "asc").select("*");
-const drain = (streamId: string) => service.drainStream({ streamId, orgId: ORG_ID, provider: LogProvider.Custom });
+const drain = (streamId: string) => service.drainStream({ streamId, orgId, provider: LogProvider.Custom });
-const cleanup = async () => {
- await testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).del();
- if (createdStreamIds.length > 0) {
- await testDb(TableName.AuditLogStreamOutbox).whereIn("streamId", createdStreamIds).del();
- await testDb(TableName.AuditLogStream).whereIn("id", createdStreamIds).del();
- createdStreamIds.length = 0;
- }
-};
+const cleanupEvents = () => testDb(TableName.EventOutbox).where({ consumer: CONSUMER }).del();
describe("audit log stream outbox (postgres)", () => {
beforeAll(async () => {
@@ -132,10 +129,14 @@ describe("audit log stream outbox (postgres)", () => {
// outbox service and the providers read have to be initialized here as well.
initLogger();
await initEnvConfig(testHsmService, testKmsRootConfigDAL, testSuperAdminDAL, logger);
- await cleanup();
+ ({ orgId, cleanup: cleanupOrg } = await createIsolatedOrgAndProject("audit-log-stream-outbox-e2e"));
+ await cleanupEvents();
+ });
+ afterEach(cleanupEvents);
+ afterAll(async () => {
+ await cleanupEvents();
+ await cleanupOrg();
});
- afterEach(cleanup);
- afterAll(cleanup);
describe("stream health and the alert event commit with the drop", () => {
test("markStreamFailing flips a healthy stream exactly once under concurrent workers", async () => {
@@ -175,7 +176,7 @@ describe("audit log stream outbox (postgres)", () => {
await emitAuditLogStreamDeliveryFailed(
eventEmitter,
{
- orgId: ORG_ID,
+ orgId,
streamId: stream.id,
provider: LogProvider.Custom,
errorMessage: "boom",
@@ -254,7 +255,7 @@ describe("audit log stream outbox (postgres)", () => {
expect(emitted).toHaveLength(1);
expect(emitted[0].eventType).toBe(AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT);
expect(emitted[0].payload).toMatchObject({
- orgId: ORG_ID,
+ orgId,
resourceType: "audit-log.stream",
resourceId: null,
targetIds: [stream.id],
diff --git a/backend/e2e-test/fakes/audit-log-stream-outbox-constants.ts b/backend/e2e-test/fakes/audit-log-stream-outbox-constants.ts
new file mode 100644
index 0000000000..d0918d91a7
--- /dev/null
+++ b/backend/e2e-test/fakes/audit-log-stream-outbox-constants.ts
@@ -0,0 +1,44 @@
+import * as RealConstants from "../../src/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-constants";
+
+// Replaces the outbox's debounce and retry limit so a spec can watch a stream fail without waiting
+// out the real timings (a 5s debounce, then five attempts spread over backoff). Wired up by
+// test.alias in vitest.e2e.config.mts; nothing under src/ references this file.
+//
+// Defaults are the real values, so specs that do not ask for anything else see production
+// behavior. The values are live bindings: the service reads them at call time, so a change made
+// through the handle below applies to the next flush.
+//
+// The handle hangs off globalThis for the reason given in aws-parameter-store-sync-fns.ts.
+/* eslint-disable import/no-mutable-exports -- a live binding is what lets the service see a change */
+export let FLUSH_DEBOUNCE_MS = RealConstants.FLUSH_DEBOUNCE_MS as number;
+export let MAX_ATTEMPTS = RealConstants.MAX_ATTEMPTS as number;
+/* eslint-enable import/no-mutable-exports */
+
+const globalScope = globalThis as typeof globalThis & {
+ infisicalAuditLogStreamOutboxTuning?: {
+ set: (opts: { flushDebounceMs?: number; maxAttempts?: number }) => void;
+ reset: () => void;
+ };
+};
+
+const reset = () => {
+ FLUSH_DEBOUNCE_MS = RealConstants.FLUSH_DEBOUNCE_MS;
+ MAX_ATTEMPTS = RealConstants.MAX_ATTEMPTS;
+};
+
+globalScope.infisicalAuditLogStreamOutboxTuning ??= {
+ set: (opts) => {
+ FLUSH_DEBOUNCE_MS = opts.flushDebounceMs ?? FLUSH_DEBOUNCE_MS;
+ MAX_ATTEMPTS = opts.maxAttempts ?? MAX_ATTEMPTS;
+ },
+ reset
+};
+
+export const auditLogStreamOutboxTuning = {
+ set: (opts: { flushDebounceMs?: number; maxAttempts?: number }) => {
+ globalScope.infisicalAuditLogStreamOutboxTuning?.set(opts);
+ },
+ reset: () => {
+ globalScope.infisicalAuditLogStreamOutboxTuning?.reset();
+ }
+};
diff --git a/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.ts b/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.ts
new file mode 100644
index 0000000000..60de388925
--- /dev/null
+++ b/backend/e2e-test/routes/v1/audit-log-stream-failure-alert.spec.ts
@@ -0,0 +1,113 @@
+import { auditLogStreamOutboxTuning } from "e2e-test/fakes/audit-log-stream-outbox-constants";
+import { eventOutbox } from "e2e-test/fakes/event-outbox-queue";
+import { createFakeWebhookServer, TFakeWebhookServer } from "e2e-test/fakes/webhook-destination";
+import { TTestSmtpService } from "e2e-test/mocks/smtp";
+import { createAlert } from "e2e-test/testUtils/alerts";
+import { createIsolatedOrgAndProject } from "e2e-test/testUtils/fixtures";
+import { overrideLicenseFeatures } from "e2e-test/testUtils/license";
+import { pollUntil } from "e2e-test/testUtils/poll";
+import { createSecretV2 } from "e2e-test/testUtils/secrets";
+import { addUserMembership, createUser, deleteUsers } from "e2e-test/testUtils/users";
+
+import { OrgMembershipRole } from "@app/db/schemas";
+import {
+ AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,
+ AUDIT_LOG_STREAM_RESOURCE_TYPE
+} from "@app/ee/services/audit-log-stream/audit-log-stream-events";
+import { ALERT_EVENT_CONSUMER } from "@app/services/alert/alert-event-consumer";
+
+const smtp = () => (globalThis as unknown as { testSmtp: TTestSmtpService }).testSmtp;
+
+// The one end-to-end check that a stream which cannot deliver ends in a notification. It goes
+// through the public API from stream creation to the alert email; the edge cases (rollback,
+// repeat failures, recovery) are covered in audit-log-stream-outbox.integration.spec.ts.
+//
+// The debounce and retry limit are shortened by a fake (see fakes/audit-log-stream-outbox-constants.ts)
+// so the first failed delivery exhausts the row, instead of waiting out five backed-off attempts.
+describe("Audit log stream failure alert", () => {
+ let orgId: string;
+ let projectId: string;
+ let authToken: string;
+ let cleanup: () => Promise<void>;
+ let restoreLicense: () => void;
+ let streamReceiver: TFakeWebhookServer;
+ let recipient: { userId: string; username: string };
+
+ beforeAll(async () => {
+ restoreLicense = overrideLicenseFeatures({
+ auditLogs: true,
+ auditLogsRetentionDays: 30,
+ auditLogStreams: true
+ });
+ auditLogStreamOutboxTuning.set({ flushDebounceMs: 1_000, maxAttempts: 1 });
+ streamReceiver = await createFakeWebhookServer();
+ ({ orgId, projectId, authToken, cleanup } = await createIsolatedOrgAndProject("audit-log-stream-failure-alert"));
+ recipient = await createUser("stream-alert-recipient");
+ await addUserMembership({ userId: recipient.userId, orgId, role: OrgMembershipRole.Member });
+ });
+
+ afterAll(async () => {
+ await cleanup();
+ await deleteUsers([recipient.userId]);
+ await streamReceiver.stop();
+ auditLogStreamOutboxTuning.reset();
+ restoreLicense();
+ });
+
+ test("a stream whose destination keeps failing alerts the org", async () => {
+ const streamRes = await testServer.inject({
+ method: "POST",
+ url: "/api/v1/audit-log-streams/custom",
+ headers: { authorization: `Bearer ${authToken}` },
+ body: { credentials: { url: streamReceiver.url, headers: [] } }
+ });
+ expect(streamRes.statusCode, streamRes.payload).toBe(200);
+ const streamId = streamRes.json().auditLogStream.id as string;
+
+ // Creating a stream sends a test request, so the destination only starts failing afterwards.
+ streamReceiver.reset();
+ streamReceiver.setStatusCode(500);
+
+ await createAlert({
+ authToken,
+ body: {
+ name: "stream failure",
+ resourceType: AUDIT_LOG_STREAM_RESOURCE_TYPE,
+ eventType: AUDIT_LOG_STREAM_DELIVERY_FAILED_EVENT,
+ channels: [
+ {
+ name: "Email",
+ channelType: "email",
+ config: {},
+ recipients: [{ principalType: "user", principalId: recipient.userId }]
+ }
+ ]
+ }
+ });
+
+ await createSecretV2({
+ workspaceId: projectId,
+ environmentSlug: "dev",
+ secretPath: "/",
+ key: "TRIGGER_AUDIT_LOG",
+ value: "value",
+ authToken
+ });
+
+ // The stream's first flush fails and drops the row. The alert event it emits is then delivered
+ // by draining the event outbox here, rather than waiting for the relay's next tick.
+ const [email] = await pollUntil({
+ describe: "the delivery-failed alert email",
+ read: async () => {
+ await eventOutbox.drain(ALERT_EVENT_CONSUMER);
+ return smtp()
+ .getEmails()
+ .filter((mail) => JSON.stringify(mail.substitutions).includes(streamId));
+ },
+ done: (emails) => emails.length > 0,
+ timeoutMs: 30_000
+ });
+
+ expect(email.recipients).toEqual([recipient.username]);
+ }, 40_000);
+});
diff --git a/backend/e2e-test/setup/reset-fakes.ts b/backend/e2e-test/setup/reset-fakes.ts
index d05c5951f2..fd7f32e7de 100644
--- a/backend/e2e-test/setup/reset-fakes.ts
+++ b/backend/e2e-test/setup/reset-fakes.ts
@@ -1,3 +1,4 @@
+import { auditLogStreamOutboxTuning } from "e2e-test/fakes/audit-log-stream-outbox-constants";
import { fakeAwsConnection } from "e2e-test/fakes/aws-connection-fns";
import { fakeIamUserSecret } from "e2e-test/fakes/aws-iam-user-secret-rotation-fns";
import { fakeParameterStore } from "e2e-test/fakes/aws-parameter-store-sync-fns";
@@ -18,4 +19,5 @@ beforeAll(() => {
fakeAwsConnection.reset();
fakeS3Bucket.reset();
fakeIamUserSecret.reset();
+ auditLogStreamOutboxTuning.reset();
});
diff --git a/backend/e2e-test/testUtils/license.ts b/backend/e2e-test/testUtils/license.ts
new file mode 100644
index 0000000000..2bba54ff74
--- /dev/null
+++ b/backend/e2e-test/testUtils/license.ts
@@ -0,0 +1,17 @@
+import type { TFeatureSet } from "@app/ee/services/license/license-types";
+
+// Turns plan features on for the whole server until the returned function is called, for a spec
+// that exercises a paid feature. The e2e license is a fixed default (see the ./license-fns alias in
+// vitest.e2e.config.mts) and getPlan hands out this one object, so the override is not scoped to an
+// org: call the restore in afterAll, and keep specs that read it in the same file.
+export const overrideLicenseFeatures = (features: Partial<TFeatureSet>) => {
+ const { license } = (globalThis as unknown as { testServices: { license: { onPremFeatures: TFeatureSet } } })
+ .testServices;
+ const plan = license.onPremFeatures as unknown as Record<string, unknown>;
+ const original = Object.fromEntries(Object.keys(features).map((key) => [key, plan[key]]));
+ Object.assign(plan, features);
+
+ return () => {
+ Object.assign(plan, original);
+ };
+};
diff --git a/backend/vitest.e2e.config.mts b/backend/vitest.e2e.config.mts
index 957dd0e859..b8cb8ffa3f 100644
--- a/backend/vitest.e2e.config.mts
+++ b/backend/vitest.e2e.config.mts
@@ -98,6 +98,11 @@ export default defineConfig({
{
find: /^@app\/services\/event-outbox\/event-outbox-queue$/,
replacement: path.resolve(__dirname, "./e2e-test/fakes/event-outbox-queue")
+ },
+ // Only audit-log-stream-outbox-service.ts imports this specifier.
+ {
+ find: "./audit-log-stream-outbox-constants",
+ replacement: path.resolve(__dirname, "./e2e-test/fakes/audit-log-stream-outbox-constants")
}
]
},