πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): fail fast and auto-disable the blob export on config faults (#16261)
Niklas Semmlerc54fa092778b
diff --git a/packages/shared/src/server/notifications/types.ts b/packages/shared/src/server/notifications/types.tsindex 1b3ccd2b05..1f45823dc5 100644--- a/packages/shared/src/server/notifications/types.ts+++ b/packages/shared/src/server/notifications/types.ts@@ -46,8 +46,8 @@ export const ProjectNotificationEventSchema = z.discriminatedUnion(eventType: z.literal(ProjectNotificationEventTypeSchema.enum["blob-export-failed"],),- // true when the integration was auto-disabled after repeated failures- // (terminal event; selects the "disabled" email variant).+ // true when the integration was auto-disabled after a configuration+ // fault (terminal event; selects the "disabled" email variant).disabled: z.boolean().optional(),}),projectNotificationEventBaseSchema.extend({diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsxindex 4ddc49de72..4ce49646b5 100644--- a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx+++ b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx@@ -16,9 +16,9 @@ export const blobStorageExportFailedEmailCopy: IntegrationExportFailedEmailCopydisabledBody: (projectName) => (<>The blob storage export for project "{projectName}- " has been disabled after repeated failures. This usually means the- destination configuration or credentials are no longer valid. Once you- have updated them, simply re-enable the export in the integration+ " has been disabled after a configuration fault. This usually means+ the destination configuration or credentials are no longer valid. Once+ you have updated them, simply re-enable the export in the integrationsettings and it will pick up right where it left off.</>),diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.tsindex 0ab4163a65..66391cbace 100644--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts@@ -238,8 +238,8 @@ describe("BlobStorageIntegrationProcessingJob", () => {});});- // After BullMQ exhausts its retries, a customer-fault error disables the- // integration; everything else keeps retrying as before.+ // A classified customer fault disables the integration on its first+ // occurrence and resolves the job; everything else keeps retrying as before.describe("customer-fault disable", () => {const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;@@ -292,28 +292,38 @@ describe("BlobStorageIntegrationProcessingJob", () => {const settleBackgroundTasks = () =>new Promise((resolve) => setTimeout(resolve, 200));- it("keeps the integration enabled and does not notify on a non-final customer-fault attempt", async () => {+ // Resolving is the point: the queue's failure counter is incremented per+ // failed attempt, so a throw here would light the processing-failures+ // monitor for a fault that no retry can clear.+ it("disables the integration and resolves on the first customer-fault attempt", async () => {const { projectId } = await createOrgProjectAndApiKey();s3Prefix = projectId;await createIntegration(projectId);+ mockRecordIncrement.mockClear();mockValidateBlobStorageEndpoint.mockRejectedValueOnce(accessDeniedError(),);- await expect(runAttempt(projectId, 0)).rejects.toThrow(/access denied/i);- await settleBackgroundTasks();+ await expect(runAttempt(projectId, 0)).resolves.toBeUndefined();const row = await prisma.blobStorageIntegration.findUniqueOrThrow({where: { projectId },});- // BullMQ still has retries left, so we let it retry β no disable yet,- // and no email either: a later retry may still succeed.- expect(row.enabled).toBe(true);+ expect(row.enabled).toBe(false);expect(row.lastError).toMatch(/access denied/i);+ expect(row.lastErrorAt).not.toBeNull();+ // The "disabled" email bypasses the cooldown, so it must not claim it.expect(row.lastFailureNotificationSentAt).toBeNull();+ // Tagged by reason so an SSRF/abuse disable stays separable from a+ // misconfiguration one.+ expect(mockRecordIncrement).toHaveBeenCalledWith(+ BLOB_INTEGRATION_DISABLED_METRIC,+ 1,+ { reason: "credentials" },+ );});- it("disables the integration on the final exhausted customer-fault attempt", async () => {+ it("still disables and resolves when the fault first appears on the final attempt", async () => {const { projectId } = await createOrgProjectAndApiKey();s3Prefix = projectId;await createIntegration(projectId);@@ -321,16 +331,12 @@ describe("BlobStorageIntegrationProcessingJob", () => {accessDeniedError(),);- await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);- await settleBackgroundTasks();+ await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();const row = await prisma.blobStorageIntegration.findUniqueOrThrow({where: { projectId },});expect(row.enabled).toBe(false);- expect(row.lastError).toMatch(/access denied/i);- expect(row.lastErrorAt).not.toBeNull();- // The "disabled" email bypasses the cooldown, so it must not claim it.expect(row.lastFailureNotificationSentAt).toBeNull();});@@ -425,7 +431,7 @@ describe("BlobStorageIntegrationProcessingJob", () => {throw accessDeniedError();});- await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);+ await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();await settleBackgroundTasks();const row = await prisma.blobStorageIntegration.findUniqueOrThrow({@@ -445,11 +451,11 @@ describe("BlobStorageIntegrationProcessingJob", () => {});// When the failure can't even be written down (Postgres unavailable), the- // run knows nothing about the row's state. It must not claim a disable it- // failed to persist, but it must still say something rather than dropping- // the failure silently. `update` is only called on this terminal path, so- // spying on it isolates the persistence failure from the export itself.- it("falls back to the informational email when persisting the failure fails", async () => {+ // run knows nothing about the row's state, so it must not claim the fault is+ // handled: it stays on the retry path and sends no email. `update` is only+ // called on this terminal path, so spying on it isolates the persistence+ // failure from the export itself.+ it("keeps retrying without notifying when persisting the failure fails", async () => {const { projectId } = await createOrgProjectAndApiKey();s3Prefix = projectId;await createIntegration(projectId);@@ -474,8 +480,8 @@ describe("BlobStorageIntegrationProcessingJob", () => {where: { projectId },});// Nothing was written, so the fault is invisible to the row and the- // integration keeps running β even though this was a classified fault on- // the final attempt, which would otherwise have disabled it.+ // integration keeps running β even though this was a classified fault+ // that would otherwise have disabled it.expect(row.enabled).toBe(true);expect(row.lastError).toBeNull();expect(mockRecordIncrement).not.toHaveBeenCalledWith(@@ -483,9 +489,9 @@ describe("BlobStorageIntegrationProcessingJob", () => {expect.anything(),expect.anything(),);- // The informational email still goes out, observable via its cooldown- // claim, so an unwritable failure is not a silent one.- expect(row.lastFailureNotificationSentAt).not.toBeNull();+ // No email either: its "will retry at the next scheduled export" text is+ // the only thing we can still vouch for, and the rethrow delivers that.+ expect(row.lastFailureNotificationSentAt).toBeNull();});});diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex d93e66d032..26b9060456 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -1520,44 +1520,40 @@ export const handleBlobStorageIntegrationProjectJob = async (} catch (error) {const errorMessage = extractStorageErrorMessage(error);- // A deterministic customer-config/credential fault can't succeed until the- // customer fixes it. Once BullMQ exhausts its retries, disable the- // integration so it stops re-scheduling and spamming.const isFinalAttempt = isFinalBullmqAttempt(job, error);- // Defined => disable; also tags the disable log + metric.+ // Defined => disable, on the first occurrence: a config fault cannot succeed+ // on a retry, and the failures metric counts every failed attempt.const customerFaultReason = classifyCustomerFault(error);const outcome = await recordTerminalExportError({projectId,errorMessage,- disableReason: isFinalAttempt ? customerFaultReason : undefined,+ disableReason: customerFaultReason,});if (outcome.kind === "integration-deleted") {return; // obsolete job: complete it rather than fail it}- // Only the informational email waits for the retries to run out; sent- // earlier it fires even when a later attempt succeeds. A disable is already- // terminal, so it ignores the attempt count.switch (outcome.kind) {case "disabled-by-us":- notifyBlobStorageExportFailedInBackground(projectId, true);- break;+ // Awaited: the integration is now off, so the scheduler never revisits+ // it and nothing would carry an interrupted dispatch.+ await notifyBlobStorageExportFailed(projectId, true);+ return; // resolving is the point; a throw would light the monitorcase "lost-disable-race":- break; // the winner sent the terminal email; ours would contradict it+ return; // the winner sent the terminal emailcase "error-recorded":// Skipped once the integration is off: "will retry at the next- // scheduled export" is no longer true.+ // scheduled export" is no longer true. Not awaited: this path rethrows,+ // so the job is about to fail and be retried regardless.if (isFinalAttempt && outcome.stillEnabled) {- notifyBlobStorageExportFailedInBackground(projectId, false);+ notifyBlobStorageExportFailed(projectId, false);}break;case "persist-failed":- // Row state unknown, so say something rather than nothing.- if (isFinalAttempt) {- notifyBlobStorageExportFailedInBackground(projectId, false);- }+ // Nothing was written, so we cannot claim the fault is handled: retry+ // and reclassify. No email β "will retry" is all we could honestly say.break;// A plain switch over a union is not exhaustiveness-checked; the never// assignment below is what makes a missing case fail to compile.@@ -1655,95 +1651,96 @@ async function recordTerminalExportError({}}-function notifyBlobStorageExportFailedInBackground(+// Logs and swallows every failure: notification trouble must not turn a+// deliberate resolve back into a job failure. Delivery is therefore best-effort+// even on the awaited path; the persisted lastError is the durable signal.+async function notifyBlobStorageExportFailed(projectId: string,disabled = false,-): void {- (async () => {- try {- // Called once per exhausted run. The cooldown gates across scheduled- // runs (the scheduler re-enqueues every frequency period, and each- // failing run would otherwise email again). The disable notification- // bypasses it: it is a one-time, terminal event β the integration- // won't run again until the customer re-enables it β and a cooldown- // claim could silently drop the one email that says it was turned off.- if (!disabled) {- const cooldownMs =- env.LANGFUSE_BLOB_STORAGE_FAILURE_NOTIFICATION_COOLDOWN_HOURS *- 60 *- 60 *- 1000;-- // Atomic claim: set timestamp before sending to prevent duplicate emails on concurrent retries.- // If the email send subsequently fails, the cooldown still applies β the next failure- // after cooldown expiry will retry the notification.- const claimed = await prisma.blobStorageIntegration.updateMany({- where: {- projectId,- OR: [- { lastFailureNotificationSentAt: null },- {- lastFailureNotificationSentAt: {- lt: new Date(Date.now() - cooldownMs),- },+): Promise<void> {+ try {+ // Called once per exhausted run. The cooldown gates across scheduled+ // runs (the scheduler re-enqueues every frequency period, and each+ // failing run would otherwise email again). The disable notification+ // bypasses it: it is a one-time, terminal event β the integration+ // won't run again until the customer re-enables it β and a cooldown+ // claim could silently drop the one email that says it was turned off.+ if (!disabled) {+ const cooldownMs =+ env.LANGFUSE_BLOB_STORAGE_FAILURE_NOTIFICATION_COOLDOWN_HOURS *+ 60 *+ 60 *+ 1000;++ // Atomic claim: set timestamp before sending to prevent duplicate emails on concurrent retries.+ // If the email send subsequently fails, the cooldown still applies β the next failure+ // after cooldown expiry will retry the notification.+ const claimed = await prisma.blobStorageIntegration.updateMany({+ where: {+ projectId,+ OR: [+ { lastFailureNotificationSentAt: null },+ {+ lastFailureNotificationSentAt: {+ lt: new Date(Date.now() - cooldownMs),},- ],- },- data: { lastFailureNotificationSentAt: new Date() },- });+ },+ ],+ },+ data: { lastFailureNotificationSentAt: new Date() },+ });- if (claimed.count === 0) {- logger.info(- `[BLOB INTEGRATION] Skipping failure notification for project ${projectId}, cooldown still active`,- );- return;- }+ if (claimed.count === 0) {+ logger.info(+ `[BLOB INTEGRATION] Skipping failure notification for project ${projectId}, cooldown still active`,+ );+ return;}+ }- const [project, integration] = await Promise.all([- prisma.project.findUnique({- where: { id: projectId },- select: { name: true },- }),- prisma.blobStorageIntegration.findUnique({- where: { projectId },- select: { bucketName: true },- }),- ]);- const projectName = project?.name ?? projectId;- const settingsPath = `/project/${projectId}/settings/integrations/blobstorage`;-- // Route to configured notification channels and admin emails. The- // cooldown claim above already deduped, so no extra throttle is needed.- // `disabled` marks the terminal "export turned off" notification, which- // selects the disabled email/subject variant downstream.- await dispatchProjectNotification({+ const [project, integration] = await Promise.all([+ prisma.project.findUnique({+ where: { id: projectId },+ select: { name: true },+ }),+ prisma.blobStorageIntegration.findUnique({+ where: { projectId },+ select: { bucketName: true },+ }),+ ]);+ const projectName = project?.name ?? projectId;+ const settingsPath = `/project/${projectId}/settings/integrations/blobstorage`;++ // Route to configured notification channels and admin emails. The+ // cooldown claim above already deduped, so no extra throttle is needed.+ // `disabled` marks the terminal "export turned off" notification, which+ // selects the disabled email/subject variant downstream.+ await dispatchProjectNotification({+ projectId,+ event: {+ eventType: "blob-export-failed",+ severity: "ALERT",projectId,- event: {- eventType: "blob-export-failed",- severity: "ALERT",- projectId,- projectName,- // The integration is keyed by projectId (1:1); the bucket name is- // the most useful human label for the failing export destination.- resourceId: projectId,- resourceName: integration?.bucketName ?? "Blob storage integration",- message: disabled- ? `Blob storage export disabled for project "${projectName}" after repeated failures.`- : `Blob storage export failed for project "${projectName}".`,- url: env.NEXTAUTH_URL- ? `${env.NEXTAUTH_URL}${settingsPath}`- : undefined,- disabled,- },- });- } catch (error) {- logger.error(- `[BLOB INTEGRATION] Failed to send failure notification for project ${projectId}`,- error,- );- }- })();+ projectName,+ // The integration is keyed by projectId (1:1); the bucket name is+ // the most useful human label for the failing export destination.+ resourceId: projectId,+ resourceName: integration?.bucketName ?? "Blob storage integration",+ message: disabled+ ? `Blob storage export disabled for project "${projectName}" after a configuration fault.`+ : `Blob storage export failed for project "${projectName}".`,+ url: env.NEXTAUTH_URL+ ? `${env.NEXTAUTH_URL}${settingsPath}`+ : undefined,+ disabled,+ },+ });+ } catch (error) {+ logger.error(+ `[BLOB INTEGRATION] Failed to send failure notification for project ${projectId}`,+ error,+ );+ }}function extractStorageErrorMessage(error: unknown): string {