Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(worker): name the blob export terminal-failure outcomes (#16257)

Niklas Semmlerbfde3e7df77f
diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
index 0decf0a41f..0ab4163a65 100644
--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
@@ -404,6 +404,89 @@ describe("BlobStorageIntegrationProcessingJob", () => {
expect(row.lastError).toMatch(/connection refused/i);
expect(row.lastFailureNotificationSentAt).toBeNull();
});
+
+ // A concurrent run (distinct jobId, so not queue-deduped) can win the
+ // disable claim first. The loser must stay silent: the winner already sent
+ // the terminal "disabled" email, and the loser's only fallback is the
+ // informational "will retry at the next scheduled export" variant, which is
+ // false for an integration that is now off.
+ it("does not notify when a concurrent run won the customer-fault disable claim", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ s3Prefix = projectId;
+ await createIntegration(projectId);
+ mockRecordIncrement.mockClear();
+ // Land the winner's disable between the handler's enabled check and the
+ // terminal catch, then raise the same fault the winner disabled on.
+ mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {
+ await prisma.blobStorageIntegration.updateMany({
+ where: { projectId },
+ data: { enabled: false },
+ });
+ throw accessDeniedError();
+ });
+
+ await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);
+ await settleBackgroundTasks();
+
+ const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
+ where: { projectId },
+ });
+ expect(row.enabled).toBe(false);
+ expect(row.lastError).toMatch(/access denied/i);
+ // Null proves the loser suppressed its email: the informational fallback
+ // would have taken the cooldown claim and stamped this.
+ expect(row.lastFailureNotificationSentAt).toBeNull();
+ // Only the worker that actually flipped enabled true->false counts it.
+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(
+ BLOB_INTEGRATION_DISABLED_METRIC,
+ expect.anything(),
+ expect.anything(),
+ );
+ });
+
+ // When the failure can't even be written down (Postgres unavailable), the
+ // run knows nothing about the row's state. It must not claim a disable it
+ // failed to persist, but it must still say something rather than dropping
+ // the failure silently. `update` is only called on this terminal path, so
+ // spying on it isolates the persistence failure from the export itself.
+ it("falls back to the informational email when persisting the failure fails", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ s3Prefix = projectId;
+ await createIntegration(projectId);
+ mockRecordIncrement.mockClear();
+ mockValidateBlobStorageEndpoint.mockRejectedValueOnce(
+ accessDeniedError(),
+ );
+ const updateSpy = vi
+ .spyOn(prisma.blobStorageIntegration, "update")
+ .mockRejectedValueOnce(new Error("database unavailable"));
+
+ try {
+ await expect(runAttempt(projectId, 4)).rejects.toThrow(
+ /access denied/i,
+ );
+ await settleBackgroundTasks();
+ } finally {
+ updateSpy.mockRestore();
+ }
+
+ const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
+ where: { projectId },
+ });
+ // Nothing was written, so the fault is invisible to the row and the
+ // integration keeps running — even though this was a classified fault on
+ // the final attempt, which would otherwise have disabled it.
+ expect(row.enabled).toBe(true);
+ expect(row.lastError).toBeNull();
+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(
+ BLOB_INTEGRATION_DISABLED_METRIC,
+ expect.anything(),
+ expect.anything(),
+ );
+ // The informational email still goes out, observable via its cooldown
+ // claim, so an unwritable failure is not a silent one.
+ expect(row.lastFailureNotificationSentAt).not.toBeNull();
+ });
});
// LFE-14894: an integration deleted mid-run makes the job obsolete — it must
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index dc1ef5506e..215d5a8de8 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -49,6 +49,7 @@ import { TimedGzip, ZLIB_DEFAULT_LEVEL, type GzipStats } from "./gzipStream";
import {
BLOB_INTEGRATION_DISABLED_METRIC,
classifyCustomerFault,
+ type CustomerFaultReason,
} from "./isCustomerFaultError";
import { isRecordNotFoundError } from "../integrations/prismaErrors";
import { ByteCounter, TimedByteCounter } from "./byteCounters";
@@ -1524,85 +1525,49 @@ export const handleBlobStorageIntegrationProjectJob = async (
// 0-based, so the final attempt is attempts - 1.
const isFinalAttempt =
(job.attemptsMade ?? 0) >= (job.opts?.attempts ?? 1) - 1;
- // The reason bucket doubles as the disable decision (defined => disable) and
- // as the tag on the disable log + metric below.
+ // Defined => disable; also tags the disable log + metric.
const customerFaultReason = classifyCustomerFault(error);
- const disableForCustomerFault =
- isFinalAttempt && customerFaultReason !== undefined;
-
- // True only for the worker that actually flips enabled true→false, so the
- // "disabled" email (which bypasses the cooldown) is sent exactly once and
- // never claims a state we failed to write. The atomic claim also dedups
- // concurrent terminal failures — e.g. a scheduled run racing a manual Run
- // Now across pods, which have distinct jobIds and so aren't queue-deduped.
- let persistedDisable = false;
- let disableClaimRan = false;
- // Assume enabled when the read-back fails; worst case is today's email.
- let enabledAfterPersist = true;
- try {
- const updated = await prisma.blobStorageIntegration.update({
- where: { projectId },
- data: {
- lastError: errorMessage,
- lastErrorAt: new Date(),
- runStartedAt: null,
- },
- });
- enabledAfterPersist = updated.enabled;
- if (disableForCustomerFault) {
- const { count } = await prisma.blobStorageIntegration.updateMany({
- where: { projectId, enabled: true },
- data: { enabled: false },
- });
- disableClaimRan = true;
- persistedDisable = count === 1;
- if (persistedDisable) {
- // Tag by reason so SSRF/abuse disables (ssrf_blocked_endpoint) can be
- // separated from customer misconfig, and a mass-disable regression is
- // visible as a spike in the non-SSRF buckets after rollout.
- const reason = customerFaultReason ?? "unknown";
- recordIncrement(BLOB_INTEGRATION_DISABLED_METRIC, 1, { reason });
- logger.warn(
- `[BLOB INTEGRATION] Disabled blob storage integration for project ${projectId} after a customer fault (reason=${reason}): ${errorMessage}`,
- { blobStorageDisableReason: reason, projectId },
- );
+
+ const outcome = await recordTerminalExportError({
+ projectId,
+ errorMessage,
+ disableReason: isFinalAttempt ? customerFaultReason : undefined,
+ });
+
+ if (outcome.kind === "integration-deleted") {
+ return; // obsolete job: complete it rather than fail it
+ }
+
+ // Only the informational email waits for the retries to run out; sent
+ // earlier it fires even when a later attempt succeeds. A disable is already
+ // terminal, so it ignores the attempt count.
+ switch (outcome.kind) {
+ case "disabled-by-us":
+ notifyBlobStorageExportFailedInBackground(projectId, true);
+ break;
+ case "lost-disable-race":
+ break; // the winner sent the terminal email; ours would contradict it
+ case "error-recorded":
+ // Skipped once the integration is off: "will retry at the next
+ // scheduled export" is no longer true.
+ if (isFinalAttempt && outcome.stillEnabled) {
+ notifyBlobStorageExportFailedInBackground(projectId, false);
}
- }
- } catch (persistError) {
- if (isRecordNotFoundError(persistError)) {
- // Integration deleted mid-run: nothing to persist, nobody to notify —
- // the notification would reference a config that no longer exists.
- // Complete the obsolete job instead of failing it.
- logger.info(
- `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${errorMessage}`,
+ break;
+ case "persist-failed":
+ // Row state unknown, so say something rather than nothing.
+ if (isFinalAttempt) {
+ notifyBlobStorageExportFailedInBackground(projectId, false);
+ }
+ break;
+ // A plain switch over a union is not exhaustiveness-checked; the never
+ // assignment below is what makes a missing case fail to compile.
+ default: {
+ const _exhaustiveCheck: never = outcome;
+ throw new Error(
+ `Unhandled terminal export outcome: ${JSON.stringify(_exhaustiveCheck)}`,
);
- return;
}
- logger.error(
- `[BLOB INTEGRATION] Failed to persist blob storage error for project ${projectId}`,
- persistError,
- );
- }
-
- // Notify only after BullMQ exhausts its retries — an email on an earlier
- // attempt fires even when a later retry succeeds, and would double up
- // with the "disabled" email on customer faults. Skip only when a
- // concurrent terminal failure won the disable claim (the winner already
- // notified); if the claim never ran because persistence threw, fall back
- // to the informational email so the failure isn't silent.
- const lostDisableRaceToConcurrentRun =
- disableForCustomerFault && disableClaimRan && !persistedDisable;
- // The informational email promises a retry at the next scheduled run,
- // which is false once the integration is disabled (a concurrent run's
- // customer-fault disable or a user toggle mid-run) — skip it then.
- const disabledOutFromUnderUs =
- !disableForCustomerFault && !enabledAfterPersist;
- if (
- isFinalAttempt &&
- !lostDisableRaceToConcurrentRun &&
- !disabledOutFromUnderUs
- ) {
- notifyBlobStorageExportFailedInBackground(projectId, persistedDisable);
}
const chain = errorChainText(error);
@@ -1620,6 +1585,77 @@ export const handleBlobStorageIntegrationProjectJob = async (
}
};
+// What the terminal bookkeeping for a failed run did. Each case is a situation
+// some earlier revision guarded against, named so the notification decision can
+// switch over them instead of recombining flags.
+type TerminalExportErrorOutcome =
+ | { kind: "integration-deleted" } // deleted mid-run: job is obsolete
+ | { kind: "persist-failed" } // write failed, so the row's state is unknown
+ // No disable attempted: not a customer fault, or retries remain. stillEnabled
+ // false means a concurrent disable or user toggle landed mid-run.
+ | { kind: "error-recorded"; stillEnabled: boolean }
+ | { kind: "disabled-by-us" }
+ | { kind: "lost-disable-race" };
+
+// Persists the failure and, when asked, makes the atomic enabled true→false
+// claim. That claim is what keeps the terminal email exactly-once across runs
+// racing the same fault (distinct jobIds, so not queue-deduped) and stops it
+// claiming a state we failed to write.
+async function recordTerminalExportError({
+ projectId,
+ errorMessage,
+ disableReason,
+}: {
+ projectId: string;
+ errorMessage: string;
+ disableReason: CustomerFaultReason | undefined;
+}): Promise<TerminalExportErrorOutcome> {
+ try {
+ const updated = await prisma.blobStorageIntegration.update({
+ where: { projectId },
+ data: {
+ lastError: errorMessage,
+ lastErrorAt: new Date(),
+ runStartedAt: null,
+ },
+ });
+
+ if (disableReason === undefined) {
+ return { kind: "error-recorded", stillEnabled: updated.enabled };
+ }
+
+ const { count } = await prisma.blobStorageIntegration.updateMany({
+ where: { projectId, enabled: true },
+ data: { enabled: false },
+ });
+ if (count !== 1) return { kind: "lost-disable-race" };
+
+ // Tag by reason so SSRF/abuse disables (ssrf_blocked_endpoint) can be
+ // separated from customer misconfig, and a mass-disable regression is
+ // visible as a spike in the non-SSRF buckets after rollout.
+ recordIncrement(BLOB_INTEGRATION_DISABLED_METRIC, 1, {
+ reason: disableReason,
+ });
+ logger.warn(
+ `[BLOB INTEGRATION] Disabled blob storage integration for project ${projectId} after a customer fault (reason=${disableReason}): ${errorMessage}`,
+ { blobStorageDisableReason: disableReason, projectId },
+ );
+ return { kind: "disabled-by-us" };
+ } catch (persistError) {
+ if (isRecordNotFoundError(persistError)) {
+ logger.info(
+ `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${errorMessage}`,
+ );
+ return { kind: "integration-deleted" };
+ }
+ logger.error(
+ `[BLOB INTEGRATION] Failed to persist blob storage error for project ${projectId}`,
+ persistError,
+ );
+ return { kind: "persist-failed" };
+ }
+}
+
function notifyBlobStorageExportFailedInBackground(
projectId: string,
disabled = false,