Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

read from primary

Thiago Araujo da Silva5d49fff0f142
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md
index 1f07821c55..846000f5b6 100644
--- a/backend/CLAUDE.md
+++ b/backend/CLAUDE.md
@@ -622,6 +622,10 @@ That's it — CRUD routes, channel creation/rotation, recipient resolution, KMS
`alertProviderRegistry.register` asserts that pairing at boot, so a provider that declares an event trigger without `findTargetsByIds` fails the process rather than silently no-op'ing in production. `triggerType` is derived from the provider's event definition inside `createAlert` and is never accepted from a request.
+**Every read on the event path goes to the primary, and `findTargetsByIds` must too.** An empty read there is terminal: "no matching alert", "no channels" and "targets gone" all mark the event delivered and nothing ever asks again, so a replica that has not caught up with a commit loses the notification outright. `findEnabledForEvent` and the engine's channel lookup (`readFromPrimary: true`) already do this; a provider's `findTargetsByIds` has to as well, because the target usually commits in the same transaction as the event that names it and the engine cannot tell "not replicated yet" from "deleted". The scheduled path keeps the replica because tomorrow's scan asks again. These reads run once per batch in the worker and are indexed point lookups, so the primary costs nothing measurable.
+
+**The history write is retried, then logged, never thrown.** By the time `createWithTargets` runs, the channels have delivered, so a throw cannot undo anything: on the event path it would re-notify on retry, and on the scheduled path it only fails the job while leaving the same dedup gap behind. A few in-process attempts are the only lever that actually narrows the gap; after that it is a `logger.error` and the run reports its delivery outcome.
+
Invariants worth knowing before extending it:
- **The alert module owns no CASL subject.** Each provider reuses its own resource's existing permissions inside `assertPermission`, so authorization stays with the domain that owns the resource.
diff --git a/backend/src/services/alert/alert-channel-dal.ts b/backend/src/services/alert/alert-channel-dal.ts
index 327765873e..ad8d546fc3 100644
--- a/backend/src/services/alert/alert-channel-dal.ts
+++ b/backend/src/services/alert/alert-channel-dal.ts
@@ -14,11 +14,11 @@ export const alertChannelDALFactory = (db: TDbClient) => {
const findByAlertId = async (
alertId: string,
- filter: { enabled?: boolean } = {},
+ filter: { enabled?: boolean; readFromPrimary?: boolean } = {},
tx?: Knex
): Promise<TAlertChannels[]> => {
try {
- const query = (tx || db.replicaNode())(TableName.AlertChannel)
+ const query = (tx || (filter.readFromPrimary ? db : db.replicaNode()))(TableName.AlertChannel)
.join(
TableName.AlertChannelMembership,
`${TableName.AlertChannel}.id`,
diff --git a/backend/src/services/alert/alert-dal.test.ts b/backend/src/services/alert/alert-dal.test.ts
index 314eab2fdd..00c3c651c4 100644
--- a/backend/src/services/alert/alert-dal.test.ts
+++ b/backend/src/services/alert/alert-dal.test.ts
@@ -67,9 +67,16 @@ const buildDAL = () => {
orderBy: async () => [] as TAlerts[]
};
const queryBuilder = () => chain;
- const db = Object.assign(queryBuilder, { replicaNode: () => queryBuilder, raw: (sql: string) => sql }) as never;
+ let replicaReads = 0;
+ const db = Object.assign(queryBuilder, {
+ replicaNode: () => {
+ replicaReads += 1;
+ return queryBuilder;
+ },
+ raw: (sql: string) => sql
+ }) as never;
- return { dal: alertDALFactory(db), calls };
+ return { dal: alertDALFactory(db), calls, getReplicaReads: () => replicaReads };
};
describe("alert dal", () => {
@@ -118,6 +125,23 @@ describe("alert dal", () => {
expect(calls.where).toContainEqual([`${TableName.Alert}.enabled`, true]);
});
+ // An empty result here is terminal for the event, so it must not come from a replica that has yet
+ // to see the alert commit.
+ test("findEnabledForEvent reads the primary while the scheduled lookup keeps the replica", async () => {
+ const { dal, getReplicaReads } = buildDAL();
+
+ await dal.findEnabledForEvent({
+ orgId: "org-1",
+ resourceType: "approval.workflow",
+ resourceId: "policy-1",
+ eventType: "approval.workflow.request_opened"
+ });
+ expect(getReplicaReads()).toBe(0);
+
+ await dal.findEnabledByResourceType("approval.workflow");
+ expect(getReplicaReads()).toBe(1);
+ });
+
test("findEnabledForEvent applies the same soft-deleted-project and enabled-channel filters", async () => {
const { dal, calls } = buildDAL();
diff --git a/backend/src/services/alert/alert-dal.ts b/backend/src/services/alert/alert-dal.ts
index ce29e706fa..bc02fe4306 100644
--- a/backend/src/services/alert/alert-dal.ts
+++ b/backend/src/services/alert/alert-dal.ts
@@ -55,7 +55,7 @@ export const alertDALFactory = (db: TDbClient) => {
tx?: Knex
): Promise<TAlerts[]> => {
try {
- const query = (tx || db.replicaNode())(TableName.Alert)
+ const query = (tx || db)(TableName.Alert)
.leftJoin(TableName.Project, `${TableName.Alert}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Alert}.orgId`, filter.orgId)
.where(`${TableName.Alert}.resourceType`, filter.resourceType)
@@ -127,7 +127,7 @@ export const alertDALFactory = (db: TDbClient) => {
tx?: Knex
): Promise<TAlerts[]> => {
try {
- const query = (tx || db.replicaNode())(TableName.Alert)
+ const query = (tx || db)(TableName.Alert)
.leftJoin(TableName.Project, `${TableName.Alert}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Alert}.orgId`, filter.orgId)
.where(`${TableName.Alert}.resourceType`, filter.resourceType)
diff --git a/backend/src/services/alert/alert-engine.test.ts b/backend/src/services/alert/alert-engine.test.ts
index c1bcf608e3..e8b7a87361 100644
--- a/backend/src/services/alert/alert-engine.test.ts
+++ b/backend/src/services/alert/alert-engine.test.ts
@@ -116,6 +116,7 @@ const buildEngine = (opts: {
failEmail?: boolean;
failEmailFor?: string;
failHistoryWrite?: boolean;
+ failHistoryWriteTimes?: number;
}) => {
const registry = alertProviderRegistryFactory();
let findDueTargetsCalls = 0;
@@ -127,6 +128,8 @@ const buildEngine = (opts: {
const sentMail: Array<{ recipients: string[] }> = [];
const historyWrites: Array<{ deliveries: TDelivery[]; status: string }> = [];
+ const channelLookups: Array<{ enabled?: boolean; readFromPrimary?: boolean } | undefined> = [];
+ let historyAttempts = 0;
// Tracks the peak number of overlapping sendMail calls so a test can assert the
// run-wide delivery concurrency stays bounded. sendMail yields a macrotask before
// resolving so concurrent sends genuinely overlap.
@@ -135,8 +138,12 @@ const buildEngine = (opts: {
const engine = alertEngineFactory({
alertChannelDAL: {
- findByAlertId: async (_alertId: string, filter?: { enabled?: boolean }) =>
- filter?.enabled === undefined ? opts.channels : opts.channels.filter((c) => c.enabled === filter.enabled)
+ findByAlertId: async (_alertId: string, filter?: { enabled?: boolean; readFromPrimary?: boolean }) => {
+ channelLookups.push(filter);
+ return filter?.enabled === undefined
+ ? opts.channels
+ : opts.channels.filter((c) => c.enabled === filter.enabled);
+ }
},
alertChannelRecipientDAL: {
// One recipient row per directed channel in the run, so each resolves its own list.
@@ -153,7 +160,10 @@ const buildEngine = (opts: {
alertHistoryDAL: {
findRecentlyAlertedTargets: async () => opts.recentlyAlerted ?? [],
createWithTargets: async (_alertId: string, options: { status: string }, deliveries: TDelivery[]) => {
- if (opts.failHistoryWrite) throw new Error("history table unavailable");
+ historyAttempts += 1;
+ if (opts.failHistoryWrite || historyAttempts <= (opts.failHistoryWriteTimes ?? 0)) {
+ throw new Error("history table unavailable");
+ }
historyWrites.push({ deliveries, status: options.status });
return {} as never;
}
@@ -188,6 +198,8 @@ const buildEngine = (opts: {
engine,
sentMail,
historyWrites,
+ channelLookups,
+ getHistoryAttempts: () => historyAttempts,
getPeakConcurrentSends: () => peakConcurrentSends,
getFindDueTargetsCalls: () => findDueTargetsCalls
};
@@ -592,6 +604,37 @@ describe("alert engine, event path", () => {
expect(sentMail).toHaveLength(1);
});
+ // The history row is the dedup record and the event path's skip list. Once the channels have sent,
+ // a retry of the insert is the only lever left that narrows the gap a transient failure opens.
+ test("retries a failed history write and records it on a later attempt", async () => {
+ const { engine, historyWrites, getHistoryAttempts } = buildEngine({
+ targets: [{ id: "t1" }],
+ channels: [{ id: "c-email", channelType: "email", encryptedConfig: encConfig({}), enabled: true }],
+ failHistoryWriteTimes: 1
+ });
+
+ const result = await engine.runAlertForEvent(eventAlert(), EVENT);
+
+ expect(result.outcome).toBe(AlertDispatchOutcome.DeliverySuccess);
+ expect(getHistoryAttempts()).toBe(2);
+ expect(historyWrites).toHaveLength(1);
+ });
+
+ // An empty channel read on the event path marks the event delivered for good, so it must not be
+ // answered by a replica that has yet to see the channel commit.
+ test("reads channels from the primary on the event path and from the replica on the scheduled one", async () => {
+ const { engine, channelLookups } = buildEngine({
+ targets: [{ id: "t1" }],
+ channels: [{ id: "c-email", channelType: "email", encryptedConfig: encConfig({}), enabled: true }]
+ });
+
+ await engine.runAlertForEvent(eventAlert(), EVENT);
+ await engine.runAlert(makeAlert());
+
+ expect(channelLookups[0]).toEqual({ enabled: true, readFromPrimary: true });
+ expect(channelLookups[1]).toEqual({ enabled: true });
+ });
+
test("reports failure and no delivered channels when every channel fails", async () => {
const { engine } = buildEngine({
targets: [{ id: "t1" }],
diff --git a/backend/src/services/alert/alert-engine.ts b/backend/src/services/alert/alert-engine.ts
index 611b3c6380..4f1ae23289 100644
--- a/backend/src/services/alert/alert-engine.ts
+++ b/backend/src/services/alert/alert-engine.ts
@@ -18,6 +18,14 @@ import { ALERT_CHANNEL_REGISTRY } from "./channels/alert-channel-registry";
const ALERT_DELIVERY_CONCURRENCY = 10;
+const HISTORY_WRITE_ATTEMPTS = 3;
+const HISTORY_WRITE_RETRY_DELAY_MS = 250;
+
+const sleep = (ms: number) =>
+ new Promise<void>((resolve) => {
+ setTimeout(resolve, ms);
+ });
+
type TChannelWork = {
channel: TAlertChannels;
due: { target: unknown; id: string }[];
@@ -189,10 +197,20 @@ export const alertEngineFactory = ({
// Deliberately not allowed to fail the run: the event path may retry, and a throw after channels
// have already sent would re-notify.
- try {
- await alertHistoryDAL.createWithTargets(alert.id, { status }, deliveries);
- } catch (err) {
- logger.error(err, `Failed to record alert history after delivery [alertId=${alert.id}]`);
+ for (let attempt = 1; attempt <= HISTORY_WRITE_ATTEMPTS; attempt += 1) {
+ try {
+ // eslint-disable-next-line no-await-in-loop -- retrying the same insert is the point
+ await alertHistoryDAL.createWithTargets(alert.id, { status }, deliveries);
+ break;
+ } catch (err) {
+ if (attempt === HISTORY_WRITE_ATTEMPTS) {
+ logger.error(err, `Failed to record alert history after delivery [alertId=${alert.id}]`);
+ break;
+ }
+ logger.warn(err, `Failed to record alert history, retrying [attempt=${attempt}] [alertId=${alert.id}]`);
+ // eslint-disable-next-line no-await-in-loop -- see above
+ await sleep(HISTORY_WRITE_RETRY_DELAY_MS * attempt);
+ }
}
const deliveredChannelIds = dispatched.filter((result) => result.success).map((result) => result.channelId);
@@ -280,7 +298,7 @@ export const alertEngineFactory = ({
}
const skip = new Set(input.skipChannelIds ?? []);
- const channels = (await alertChannelDAL.findByAlertId(alert.id, { enabled: true })).filter(
+ const channels = (await alertChannelDAL.findByAlertId(alert.id, { enabled: true, readFromPrimary: true })).filter(
(channel) => !skip.has(channel.id)
);
if (channels.length === 0) return { outcome: AlertDispatchOutcome.NoChannels, deliveredChannelIds: [] };
diff --git a/backend/src/services/alert/alert-types.ts b/backend/src/services/alert/alert-types.ts
index 033d324b8b..e955952e4e 100644
--- a/backend/src/services/alert/alert-types.ts
+++ b/backend/src/services/alert/alert-types.ts
@@ -118,6 +118,9 @@ export interface IResourceAlertProvider<TTarget = unknown> {
// Rehydrate the targets an event named, by id. Nothing is scanned for: this exists so the row is
// read at delivery time rather than copied into the event. A target whose row is gone was deleted
// between emit and dispatch, so it's dropped rather than an error.
+ // Must read from the primary, never a replica: the target usually commits in the same transaction
+ // as the event that names it, and the engine cannot tell "not replicated yet" from "deleted". An
+ // empty result is terminal for the event, so a stale replica here loses the notification outright.
// Required for any Event event; the registry enforces that at boot.
findTargetsByIds?(input: TFindTargetsByIdsInput): Promise<TTarget[]>;