🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(worker): name the blob export terminal-failure outcomes (#16257)
Niklas Semmlerbfde3e7df77f
diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.tsindex 0decf0a41f..0ab4163a65 100644--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts@@ -404,6 +404,89 @@ describe("BlobStorageIntegrationProcessingJob", () => {expect(row.lastError).toMatch(/connection refused/i);expect(row.lastFailureNotificationSentAt).toBeNull();});++ // A concurrent run (distinct jobId, so not queue-deduped) can win the+ // disable claim first. The loser must stay silent: the winner already sent+ // the terminal "disabled" email, and the loser's only fallback is the+ // informational "will retry at the next scheduled export" variant, which is+ // false for an integration that is now off.+ it("does not notify when a concurrent run won the customer-fault disable claim", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ s3Prefix = projectId;+ await createIntegration(projectId);+ mockRecordIncrement.mockClear();+ // Land the winner's disable between the handler's enabled check and the+ // terminal catch, then raise the same fault the winner disabled on.+ mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {+ await prisma.blobStorageIntegration.updateMany({+ where: { projectId },+ data: { enabled: false },+ });+ throw accessDeniedError();+ });++ await expect(runAttempt(projectId, 4)).rejects.toThrow(/access denied/i);+ await settleBackgroundTasks();++ const row = await prisma.blobStorageIntegration.findUniqueOrThrow({+ where: { projectId },+ });+ expect(row.enabled).toBe(false);+ expect(row.lastError).toMatch(/access denied/i);+ // Null proves the loser suppressed its email: the informational fallback+ // would have taken the cooldown claim and stamped this.+ expect(row.lastFailureNotificationSentAt).toBeNull();+ // Only the worker that actually flipped enabled true->false counts it.+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(+ BLOB_INTEGRATION_DISABLED_METRIC,+ expect.anything(),+ expect.anything(),+ );+ });++ // When the failure can't even be written down (Postgres unavailable), the+ // run knows nothing about the row's state. It must not claim a disable it+ // failed to persist, but it must still say something rather than dropping+ // the failure silently. `update` is only called on this terminal path, so+ // spying on it isolates the persistence failure from the export itself.+ it("falls back to the informational email when persisting the failure fails", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ s3Prefix = projectId;+ await createIntegration(projectId);+ mockRecordIncrement.mockClear();+ mockValidateBlobStorageEndpoint.mockRejectedValueOnce(+ accessDeniedError(),+ );+ const updateSpy = vi+ .spyOn(prisma.blobStorageIntegration, "update")+ .mockRejectedValueOnce(new Error("database unavailable"));++ try {+ await expect(runAttempt(projectId, 4)).rejects.toThrow(+ /access denied/i,+ );+ await settleBackgroundTasks();+ } finally {+ updateSpy.mockRestore();+ }++ const row = await prisma.blobStorageIntegration.findUniqueOrThrow({+ where: { projectId },+ });+ // Nothing was written, so the fault is invisible to the row and the+ // integration keeps running — even though this was a classified fault on+ // the final attempt, which would otherwise have disabled it.+ expect(row.enabled).toBe(true);+ expect(row.lastError).toBeNull();+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(+ BLOB_INTEGRATION_DISABLED_METRIC,+ expect.anything(),+ expect.anything(),+ );+ // The informational email still goes out, observable via its cooldown+ // claim, so an unwritable failure is not a silent one.+ expect(row.lastFailureNotificationSentAt).not.toBeNull();+ });});// LFE-14894: an integration deleted mid-run makes the job obsolete — it mustdiff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex dc1ef5506e..215d5a8de8 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -49,6 +49,7 @@ import { TimedGzip, ZLIB_DEFAULT_LEVEL, type GzipStats } from "./gzipStream";import {BLOB_INTEGRATION_DISABLED_METRIC,classifyCustomerFault,+ type CustomerFaultReason,} from "./isCustomerFaultError";import { isRecordNotFoundError } from "../integrations/prismaErrors";import { ByteCounter, TimedByteCounter } from "./byteCounters";@@ -1524,85 +1525,49 @@ export const handleBlobStorageIntegrationProjectJob = async (// 0-based, so the final attempt is attempts - 1.const isFinalAttempt =(job.attemptsMade ?? 0) >= (job.opts?.attempts ?? 1) - 1;- // The reason bucket doubles as the disable decision (defined => disable) and- // as the tag on the disable log + metric below.+ // Defined => disable; also tags the disable log + metric.const customerFaultReason = classifyCustomerFault(error);- const disableForCustomerFault =- isFinalAttempt && customerFaultReason !== undefined;-- // True only for the worker that actually flips enabled true→false, so the- // "disabled" email (which bypasses the cooldown) is sent exactly once and- // never claims a state we failed to write. The atomic claim also dedups- // concurrent terminal failures — e.g. a scheduled run racing a manual Run- // Now across pods, which have distinct jobIds and so aren't queue-deduped.- let persistedDisable = false;- let disableClaimRan = false;- // Assume enabled when the read-back fails; worst case is today's email.- let enabledAfterPersist = true;- try {- const updated = await prisma.blobStorageIntegration.update({- where: { projectId },- data: {- lastError: errorMessage,- lastErrorAt: new Date(),- runStartedAt: null,- },- });- enabledAfterPersist = updated.enabled;- if (disableForCustomerFault) {- const { count } = await prisma.blobStorageIntegration.updateMany({- where: { projectId, enabled: true },- data: { enabled: false },- });- disableClaimRan = true;- persistedDisable = count === 1;- if (persistedDisable) {- // Tag by reason so SSRF/abuse disables (ssrf_blocked_endpoint) can be- // separated from customer misconfig, and a mass-disable regression is- // visible as a spike in the non-SSRF buckets after rollout.- const reason = customerFaultReason ?? "unknown";- recordIncrement(BLOB_INTEGRATION_DISABLED_METRIC, 1, { reason });- logger.warn(- `[BLOB INTEGRATION] Disabled blob storage integration for project ${projectId} after a customer fault (reason=${reason}): ${errorMessage}`,- { blobStorageDisableReason: reason, projectId },- );++ const outcome = await recordTerminalExportError({+ projectId,+ errorMessage,+ disableReason: isFinalAttempt ? customerFaultReason : undefined,+ });++ if (outcome.kind === "integration-deleted") {+ return; // obsolete job: complete it rather than fail it+ }++ // Only the informational email waits for the retries to run out; sent+ // earlier it fires even when a later attempt succeeds. A disable is already+ // terminal, so it ignores the attempt count.+ switch (outcome.kind) {+ case "disabled-by-us":+ notifyBlobStorageExportFailedInBackground(projectId, true);+ break;+ case "lost-disable-race":+ break; // the winner sent the terminal email; ours would contradict it+ case "error-recorded":+ // Skipped once the integration is off: "will retry at the next+ // scheduled export" is no longer true.+ if (isFinalAttempt && outcome.stillEnabled) {+ notifyBlobStorageExportFailedInBackground(projectId, false);}- }- } catch (persistError) {- if (isRecordNotFoundError(persistError)) {- // Integration deleted mid-run: nothing to persist, nobody to notify —- // the notification would reference a config that no longer exists.- // Complete the obsolete job instead of failing it.- logger.info(- `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${errorMessage}`,+ break;+ case "persist-failed":+ // Row state unknown, so say something rather than nothing.+ if (isFinalAttempt) {+ notifyBlobStorageExportFailedInBackground(projectId, false);+ }+ break;+ // A plain switch over a union is not exhaustiveness-checked; the never+ // assignment below is what makes a missing case fail to compile.+ default: {+ const _exhaustiveCheck: never = outcome;+ throw new Error(+ `Unhandled terminal export outcome: ${JSON.stringify(_exhaustiveCheck)}`,);- return;}- logger.error(- `[BLOB INTEGRATION] Failed to persist blob storage error for project ${projectId}`,- persistError,- );- }-- // Notify only after BullMQ exhausts its retries — an email on an earlier- // attempt fires even when a later retry succeeds, and would double up- // with the "disabled" email on customer faults. Skip only when a- // concurrent terminal failure won the disable claim (the winner already- // notified); if the claim never ran because persistence threw, fall back- // to the informational email so the failure isn't silent.- const lostDisableRaceToConcurrentRun =- disableForCustomerFault && disableClaimRan && !persistedDisable;- // The informational email promises a retry at the next scheduled run,- // which is false once the integration is disabled (a concurrent run's- // customer-fault disable or a user toggle mid-run) — skip it then.- const disabledOutFromUnderUs =- !disableForCustomerFault && !enabledAfterPersist;- if (- isFinalAttempt &&- !lostDisableRaceToConcurrentRun &&- !disabledOutFromUnderUs- ) {- notifyBlobStorageExportFailedInBackground(projectId, persistedDisable);}const chain = errorChainText(error);@@ -1620,6 +1585,77 @@ export const handleBlobStorageIntegrationProjectJob = async (}};+// What the terminal bookkeeping for a failed run did. Each case is a situation+// some earlier revision guarded against, named so the notification decision can+// switch over them instead of recombining flags.+type TerminalExportErrorOutcome =+ | { kind: "integration-deleted" } // deleted mid-run: job is obsolete+ | { kind: "persist-failed" } // write failed, so the row's state is unknown+ // No disable attempted: not a customer fault, or retries remain. stillEnabled+ // false means a concurrent disable or user toggle landed mid-run.+ | { kind: "error-recorded"; stillEnabled: boolean }+ | { kind: "disabled-by-us" }+ | { kind: "lost-disable-race" };++// Persists the failure and, when asked, makes the atomic enabled true→false+// claim. That claim is what keeps the terminal email exactly-once across runs+// racing the same fault (distinct jobIds, so not queue-deduped) and stops it+// claiming a state we failed to write.+async function recordTerminalExportError({+ projectId,+ errorMessage,+ disableReason,+}: {+ projectId: string;+ errorMessage: string;+ disableReason: CustomerFaultReason | undefined;+}): Promise<TerminalExportErrorOutcome> {+ try {+ const updated = await prisma.blobStorageIntegration.update({+ where: { projectId },+ data: {+ lastError: errorMessage,+ lastErrorAt: new Date(),+ runStartedAt: null,+ },+ });++ if (disableReason === undefined) {+ return { kind: "error-recorded", stillEnabled: updated.enabled };+ }++ const { count } = await prisma.blobStorageIntegration.updateMany({+ where: { projectId, enabled: true },+ data: { enabled: false },+ });+ if (count !== 1) return { kind: "lost-disable-race" };++ // Tag by reason so SSRF/abuse disables (ssrf_blocked_endpoint) can be+ // separated from customer misconfig, and a mass-disable regression is+ // visible as a spike in the non-SSRF buckets after rollout.+ recordIncrement(BLOB_INTEGRATION_DISABLED_METRIC, 1, {+ reason: disableReason,+ });+ logger.warn(+ `[BLOB INTEGRATION] Disabled blob storage integration for project ${projectId} after a customer fault (reason=${disableReason}): ${errorMessage}`,+ { blobStorageDisableReason: disableReason, projectId },+ );+ return { kind: "disabled-by-us" };+ } catch (persistError) {+ if (isRecordNotFoundError(persistError)) {+ logger.info(+ `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${errorMessage}`,+ );+ return { kind: "integration-deleted" };+ }+ logger.error(+ `[BLOB INTEGRATION] Failed to persist blob storage error for project ${projectId}`,+ persistError,+ );+ return { kind: "persist-failed" };+ }+}+function notifyBlobStorageExportFailedInBackground(projectId: string,disabled = false,