Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): fail fast and auto-disable the blob export on config faults (#16261)

Niklas Semmlerc54fa092778b
diff --git a/packages/shared/src/server/notifications/types.ts b/packages/shared/src/server/notifications/types.ts
index 1b3ccd2b05..1f45823dc5 100644
--- a/packages/shared/src/server/notifications/types.ts
+++ b/packages/shared/src/server/notifications/types.ts
@@ -46,8 +46,8 @@ export const ProjectNotificationEventSchema = z.discriminatedUnion(
eventType: z.literal(
ProjectNotificationEventTypeSchema.enum["blob-export-failed"],
),
- // true when the integration was auto-disabled after repeated failures
- // (terminal event; selects the "disabled" email variant).
+ // true when the integration was auto-disabled after a configuration
+ // fault (terminal event; selects the "disabled" email variant).
disabled: z.boolean().optional(),
}),
projectNotificationEventBaseSchema.extend({
diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx
index 4ddc49de72..4ce49646b5 100644
--- a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx
+++ b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx
@@ -16,9 +16,9 @@ export const blobStorageExportFailedEmailCopy: IntegrationExportFailedEmailCopy
disabledBody: (projectName) => (
<>
The blob storage export for project &quot;{projectName}
- &quot; has been disabled after repeated failures. This usually means the
- destination configuration or credentials are no longer valid. Once you
- have updated them, simply re-enable the export in the integration
+ &quot; has been disabled after a configuration fault. This usually means
+ the destination configuration or credentials are no longer valid. Once
+ you have updated them, simply re-enable the export in the integration
settings and it will pick up right where it left off.
</>
),
diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
index 0ab4163a65..66391cbace 100644
--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
@@ -238,8 +238,8 @@ describe("BlobStorageIntegrationProcessingJob", () => {
});
});
- // After BullMQ exhausts its retries, a customer-fault error disables the
- // integration; everything else keeps retrying as before.
+ // A classified customer fault disables the integration on its first
+ // occurrence and resolves the job; everything else keeps retrying as before.
describe("customer-fault disable", () => {
const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
@@ -292,28 +292,38 @@ describe("BlobStorageIntegrationProcessingJob", () => {
const settleBackgroundTasks = () =>
new Promise((resolve) => setTimeout(resolve, 200));
- it("keeps the integration enabled and does not notify on a non-final customer-fault attempt", async () => {
+ // Resolving is the point: the queue's failure counter is incremented per
+ // failed attempt, so a throw here would light the processing-failures
+ // monitor for a fault that no retry can clear.
+ it("disables the integration and resolves on the first customer-fault attempt", async () => {
const { projectId } = await createOrgProjectAndApiKey();
s3Prefix = projectId;
await createIntegration(projectId);
+ mockRecordIncrement.mockClear();
mockValidateBlobStorageEndpoint.mockRejectedValueOnce(
accessDeniedError(),
);
- await expect(runAttempt(projectId, 0)).rejects.toThrow(/access denied/i);
- await settleBackgroundTasks();
+ await expect(runAttempt(projectId, 0)).resolves.toBeUndefined();
const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
where: { projectId },
});
- // BullMQ still has retries left, so we let it retry β€” no disable yet,
- // and no email either: a later retry may still succeed.
- expect(row.enabled).toBe(true);
+ expect(row.enabled).toBe(false);
expect(row.lastError).toMatch(/access denied/i);
+ expect(row.lastErrorAt).not.toBeNull();
+ // The "disabled" email bypasses the cooldown, so it must not claim it.
expect(row.lastFailureNotificationSentAt).toBeNull();
+ // Tagged by reason so an SSRF/abuse disable stays separable from a
+ // misconfiguration one.
+ expect(mockRecordIncrement).toHaveBeenCalledWith(
+ BLOB_INTEGRATION_DISABLED_METRIC,
+ 1,
+ { reason: "credentials" },
+ );
});
- it("disables the integration on the final exhausted customer-fault attempt", async () => {
+ it("still disables and resolves when the fault first appears on the final attempt", async () => {
const { projectId } = await createOrgProjectAndApiKey();
s3Prefix = projectId;
await createIntegration(projectId);
@@ -321,16 +331,12 @@ describe("BlobStorageIntegrationProcessingJob", () => {
accessDeniedError(),
);
- await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);
- await settleBackgroundTasks();
+ await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();
const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
where: { projectId },
});
expect(row.enabled).toBe(false);
- expect(row.lastError).toMatch(/access denied/i);
- expect(row.lastErrorAt).not.toBeNull();
- // The "disabled" email bypasses the cooldown, so it must not claim it.
expect(row.lastFailureNotificationSentAt).toBeNull();
});
@@ -425,7 +431,7 @@ describe("BlobStorageIntegrationProcessingJob", () => {
throw accessDeniedError();
});
- await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);
+ await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();
await settleBackgroundTasks();
const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
@@ -445,11 +451,11 @@ describe("BlobStorageIntegrationProcessingJob", () => {
});
// When the failure can't even be written down (Postgres unavailable), the
- // run knows nothing about the row's state. It must not claim a disable it
- // failed to persist, but it must still say something rather than dropping
- // the failure silently. `update` is only called on this terminal path, so
- // spying on it isolates the persistence failure from the export itself.
- it("falls back to the informational email when persisting the failure fails", async () => {
+ // run knows nothing about the row's state, so it must not claim the fault is
+ // handled: it stays on the retry path and sends no email. `update` is only
+ // called on this terminal path, so spying on it isolates the persistence
+ // failure from the export itself.
+ it("keeps retrying without notifying when persisting the failure fails", async () => {
const { projectId } = await createOrgProjectAndApiKey();
s3Prefix = projectId;
await createIntegration(projectId);
@@ -474,8 +480,8 @@ describe("BlobStorageIntegrationProcessingJob", () => {
where: { projectId },
});
// Nothing was written, so the fault is invisible to the row and the
- // integration keeps running β€” even though this was a classified fault on
- // the final attempt, which would otherwise have disabled it.
+ // integration keeps running β€” even though this was a classified fault
+ // that would otherwise have disabled it.
expect(row.enabled).toBe(true);
expect(row.lastError).toBeNull();
expect(mockRecordIncrement).not.toHaveBeenCalledWith(
@@ -483,9 +489,9 @@ describe("BlobStorageIntegrationProcessingJob", () => {
expect.anything(),
expect.anything(),
);
- // The informational email still goes out, observable via its cooldown
- // claim, so an unwritable failure is not a silent one.
- expect(row.lastFailureNotificationSentAt).not.toBeNull();
+ // No email either: its "will retry at the next scheduled export" text is
+ // the only thing we can still vouch for, and the rethrow delivers that.
+ expect(row.lastFailureNotificationSentAt).toBeNull();
});
});
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index d93e66d032..26b9060456 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -1520,44 +1520,40 @@ export const handleBlobStorageIntegrationProjectJob = async (
} catch (error) {
const errorMessage = extractStorageErrorMessage(error);
- // A deterministic customer-config/credential fault can't succeed until the
- // customer fixes it. Once BullMQ exhausts its retries, disable the
- // integration so it stops re-scheduling and spamming.
const isFinalAttempt = isFinalBullmqAttempt(job, error);
- // Defined => disable; also tags the disable log + metric.
+ // Defined => disable, on the first occurrence: a config fault cannot succeed
+ // on a retry, and the failures metric counts every failed attempt.
const customerFaultReason = classifyCustomerFault(error);
const outcome = await recordTerminalExportError({
projectId,
errorMessage,
- disableReason: isFinalAttempt ? customerFaultReason : undefined,
+ disableReason: customerFaultReason,
});
if (outcome.kind === "integration-deleted") {
return; // obsolete job: complete it rather than fail it
}
- // Only the informational email waits for the retries to run out; sent
- // earlier it fires even when a later attempt succeeds. A disable is already
- // terminal, so it ignores the attempt count.
switch (outcome.kind) {
case "disabled-by-us":
- notifyBlobStorageExportFailedInBackground(projectId, true);
- break;
+ // Awaited: the integration is now off, so the scheduler never revisits
+ // it and nothing would carry an interrupted dispatch.
+ await notifyBlobStorageExportFailed(projectId, true);
+ return; // resolving is the point; a throw would light the monitor
case "lost-disable-race":
- break; // the winner sent the terminal email; ours would contradict it
+ return; // the winner sent the terminal email
case "error-recorded":
// Skipped once the integration is off: "will retry at the next
- // scheduled export" is no longer true.
+ // scheduled export" is no longer true. Not awaited: this path rethrows,
+ // so the job is about to fail and be retried regardless.
if (isFinalAttempt && outcome.stillEnabled) {
- notifyBlobStorageExportFailedInBackground(projectId, false);
+ notifyBlobStorageExportFailed(projectId, false);
}
break;
case "persist-failed":
- // Row state unknown, so say something rather than nothing.
- if (isFinalAttempt) {
- notifyBlobStorageExportFailedInBackground(projectId, false);
- }
+ // Nothing was written, so we cannot claim the fault is handled: retry
+ // and reclassify. No email β€” "will retry" is all we could honestly say.
break;
// A plain switch over a union is not exhaustiveness-checked; the never
// assignment below is what makes a missing case fail to compile.
@@ -1655,95 +1651,96 @@ async function recordTerminalExportError({
}
}
-function notifyBlobStorageExportFailedInBackground(
+// Logs and swallows every failure: notification trouble must not turn a
+// deliberate resolve back into a job failure. Delivery is therefore best-effort
+// even on the awaited path; the persisted lastError is the durable signal.
+async function notifyBlobStorageExportFailed(
projectId: string,
disabled = false,
-): void {
- (async () => {
- try {
- // Called once per exhausted run. The cooldown gates across scheduled
- // runs (the scheduler re-enqueues every frequency period, and each
- // failing run would otherwise email again). The disable notification
- // bypasses it: it is a one-time, terminal event β€” the integration
- // won't run again until the customer re-enables it β€” and a cooldown
- // claim could silently drop the one email that says it was turned off.
- if (!disabled) {
- const cooldownMs =
- env.LANGFUSE_BLOB_STORAGE_FAILURE_NOTIFICATION_COOLDOWN_HOURS *
- 60 *
- 60 *
- 1000;
-
- // Atomic claim: set timestamp before sending to prevent duplicate emails on concurrent retries.
- // If the email send subsequently fails, the cooldown still applies β€” the next failure
- // after cooldown expiry will retry the notification.
- const claimed = await prisma.blobStorageIntegration.updateMany({
- where: {
- projectId,
- OR: [
- { lastFailureNotificationSentAt: null },
- {
- lastFailureNotificationSentAt: {
- lt: new Date(Date.now() - cooldownMs),
- },
+): Promise<void> {
+ try {
+ // Called once per exhausted run. The cooldown gates across scheduled
+ // runs (the scheduler re-enqueues every frequency period, and each
+ // failing run would otherwise email again). The disable notification
+ // bypasses it: it is a one-time, terminal event β€” the integration
+ // won't run again until the customer re-enables it β€” and a cooldown
+ // claim could silently drop the one email that says it was turned off.
+ if (!disabled) {
+ const cooldownMs =
+ env.LANGFUSE_BLOB_STORAGE_FAILURE_NOTIFICATION_COOLDOWN_HOURS *
+ 60 *
+ 60 *
+ 1000;
+
+ // Atomic claim: set timestamp before sending to prevent duplicate emails on concurrent retries.
+ // If the email send subsequently fails, the cooldown still applies β€” the next failure
+ // after cooldown expiry will retry the notification.
+ const claimed = await prisma.blobStorageIntegration.updateMany({
+ where: {
+ projectId,
+ OR: [
+ { lastFailureNotificationSentAt: null },
+ {
+ lastFailureNotificationSentAt: {
+ lt: new Date(Date.now() - cooldownMs),
},
- ],
- },
- data: { lastFailureNotificationSentAt: new Date() },
- });
+ },
+ ],
+ },
+ data: { lastFailureNotificationSentAt: new Date() },
+ });
- if (claimed.count === 0) {
- logger.info(
- `[BLOB INTEGRATION] Skipping failure notification for project ${projectId}, cooldown still active`,
- );
- return;
- }
+ if (claimed.count === 0) {
+ logger.info(
+ `[BLOB INTEGRATION] Skipping failure notification for project ${projectId}, cooldown still active`,
+ );
+ return;
}
+ }
- const [project, integration] = await Promise.all([
- prisma.project.findUnique({
- where: { id: projectId },
- select: { name: true },
- }),
- prisma.blobStorageIntegration.findUnique({
- where: { projectId },
- select: { bucketName: true },
- }),
- ]);
- const projectName = project?.name ?? projectId;
- const settingsPath = `/project/${projectId}/settings/integrations/blobstorage`;
-
- // Route to configured notification channels and admin emails. The
- // cooldown claim above already deduped, so no extra throttle is needed.
- // `disabled` marks the terminal "export turned off" notification, which
- // selects the disabled email/subject variant downstream.
- await dispatchProjectNotification({
+ const [project, integration] = await Promise.all([
+ prisma.project.findUnique({
+ where: { id: projectId },
+ select: { name: true },
+ }),
+ prisma.blobStorageIntegration.findUnique({
+ where: { projectId },
+ select: { bucketName: true },
+ }),
+ ]);
+ const projectName = project?.name ?? projectId;
+ const settingsPath = `/project/${projectId}/settings/integrations/blobstorage`;
+
+ // Route to configured notification channels and admin emails. The
+ // cooldown claim above already deduped, so no extra throttle is needed.
+ // `disabled` marks the terminal "export turned off" notification, which
+ // selects the disabled email/subject variant downstream.
+ await dispatchProjectNotification({
+ projectId,
+ event: {
+ eventType: "blob-export-failed",
+ severity: "ALERT",
projectId,
- event: {
- eventType: "blob-export-failed",
- severity: "ALERT",
- projectId,
- projectName,
- // The integration is keyed by projectId (1:1); the bucket name is
- // the most useful human label for the failing export destination.
- resourceId: projectId,
- resourceName: integration?.bucketName ?? "Blob storage integration",
- message: disabled
- ? `Blob storage export disabled for project "${projectName}" after repeated failures.`
- : `Blob storage export failed for project "${projectName}".`,
- url: env.NEXTAUTH_URL
- ? `${env.NEXTAUTH_URL}${settingsPath}`
- : undefined,
- disabled,
- },
- });
- } catch (error) {
- logger.error(
- `[BLOB INTEGRATION] Failed to send failure notification for project ${projectId}`,
- error,
- );
- }
- })();
+ projectName,
+ // The integration is keyed by projectId (1:1); the bucket name is
+ // the most useful human label for the failing export destination.
+ resourceId: projectId,
+ resourceName: integration?.bucketName ?? "Blob storage integration",
+ message: disabled
+ ? `Blob storage export disabled for project "${projectName}" after a configuration fault.`
+ : `Blob storage export failed for project "${projectName}".`,
+ url: env.NEXTAUTH_URL
+ ? `${env.NEXTAUTH_URL}${settingsPath}`
+ : undefined,
+ disabled,
+ },
+ });
+ } catch (error) {
+ logger.error(
+ `[BLOB INTEGRATION] Failed to send failure notification for project ${projectId}`,
+ error,
+ );
+ }
}
function extractStorageErrorMessage(error: unknown): string {