Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(worker,shared): share integration helpers across export integrations (#16094)

Niklas Semmler7c84312f69fa
diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx
new file mode 100644
index 0000000000..4ddc49de72
--- /dev/null
+++ b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx
@@ -0,0 +1,29 @@
+import React from "react";
+import {
+ sendIntegrationExportFailedEmail,
+ type SendIntegrationExportFailedEmailParams,
+} from "../integrationExportFailed/sendIntegrationExportFailedEmail";
+import { type IntegrationExportFailedEmailCopy } from "../integrationExportFailed/IntegrationExportFailedEmailTemplate";
+
+export type SendBlobStorageExportFailedEmailParams =
+ SendIntegrationExportFailedEmailParams;
+
+export const blobStorageExportFailedEmailCopy: IntegrationExportFailedEmailCopy =
+ {
+ subjectLabel: "Blob storage",
+ headingLabel: "Blob Storage",
+ inlineLabel: "blob storage",
+ disabledBody: (projectName) => (
+ <>
+ The blob storage export for project &quot;{projectName}
+ &quot; has been disabled after repeated failures. This usually means the
+ destination configuration or credentials are no longer valid. Once you
+ have updated them, simply re-enable the export in the integration
+ settings and it will pick up right where it left off.
+ </>
+ ),
+ };
+
+export const sendBlobStorageExportFailedEmail = (
+ params: SendBlobStorageExportFailedEmailParams,
+) => sendIntegrationExportFailedEmail(blobStorageExportFailedEmailCopy, params);
diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsx b/packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsx
similarity index 64%
rename from packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsx
rename to packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsx
index 4efffae4e9..784a11bfa7 100644
--- a/packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsx
+++ b/packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsx
@@ -1,4 +1,4 @@
-import React from "react";
+import React, { type ReactNode } from "react";
import {
Body,
Button,
@@ -14,22 +14,33 @@ import {
Text,
} from "@react-email/components";
-type BlobStorageExportFailedEmailTemplateProps = {
+export type IntegrationExportFailedEmailCopy = {
+ /** Sentence-initial, for the subject and preview: "Blob storage export failed for …". */
+ subjectLabel: string;
+ /** Title Case, for the email heading: "Blob Storage Export Disabled". */
+ headingLabel: string;
+ /** Mid-sentence, for body prose and log lines: "The scheduled blob storage export …". */
+ inlineLabel: string;
+ /** Shown when the export was turned off; the remediation is integration specific. */
+ disabledBody: (projectName: string) => ReactNode;
+};
+
+type IntegrationExportFailedEmailTemplateProps = {
+ copy: IntegrationExportFailedEmailCopy;
projectName: string;
settingsUrl: string;
- // When true, the export was turned off after repeated failures and the
- // customer must fix their config and re-enable it β€” not a transient failure.
disabled?: boolean;
};
-export const BlobStorageExportFailedEmailTemplate = ({
+export const IntegrationExportFailedEmailTemplate = ({
+ copy,
projectName,
settingsUrl,
disabled = false,
-}: BlobStorageExportFailedEmailTemplateProps) => {
+}: IntegrationExportFailedEmailTemplateProps) => {
const preview = disabled
- ? `Blob storage export disabled for project "${projectName}"`
- : `Blob storage export failed for project "${projectName}"`;
+ ? `${copy.subjectLabel} export disabled for project "${projectName}"`
+ : `${copy.subjectLabel} export failed for project "${projectName}"`;
return (
<Html>
<Head />
@@ -50,22 +61,15 @@ export const BlobStorageExportFailedEmailTemplate = ({
<Section>
<Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
{disabled
- ? "Blob Storage Export Disabled"
- : "Blob Storage Export Failed"}
+ ? `${copy.headingLabel} Export Disabled`
+ : `${copy.headingLabel} Export Failed`}
</Heading>
<Text className="text-gray-700 text-sm leading-6">
{disabled ? (
- <>
- The blob storage export for project &quot;{projectName}
- &quot; has been disabled after repeated failures. This
- usually means the destination configuration or credentials
- are no longer valid. Once you have updated them, simply
- re-enable the export in the integration settings and it will
- pick up right where it left off.
- </>
+ copy.disabledBody(projectName)
) : (
<>
- The scheduled blob storage export for project &quot;
+ The scheduled {copy.inlineLabel} export for project &quot;
{projectName}&quot; has failed after multiple attempts. It
will be retried automatically at the next scheduled export.
If the issue persists, review the integration settings to
@@ -89,7 +93,7 @@ export const BlobStorageExportFailedEmailTemplate = ({
<Section>
<Text className="text-[#666666] text-[12px] leading-[24px]">
This notification was sent to project admins regarding a{" "}
- {disabled ? "disabled" : "failed"} blob storage export for
+ {disabled ? "disabled" : "failed"} {copy.inlineLabel} export for
project &quot;{projectName}&quot;.
</Text>
</Section>
diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.ts b/packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.ts
similarity index 62%
rename from packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.ts
rename to packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.ts
index 06eef9d958..1aceb47de5 100644
--- a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.ts
+++ b/packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.ts
@@ -3,9 +3,12 @@ import { createMailTransport } from "../transport";
import { z } from "zod";
import { sanitizeEmailSubject } from "../../../../utils/zod";
import { logger } from "../../../logger";
-import { BlobStorageExportFailedEmailTemplate } from "./BlobStorageExportFailedEmailTemplate";
+import {
+ IntegrationExportFailedEmailTemplate,
+ type IntegrationExportFailedEmailCopy,
+} from "./IntegrationExportFailedEmailTemplate";
-export type SendBlobStorageExportFailedEmailParams = {
+export type SendIntegrationExportFailedEmailParams = {
env: Partial<
Record<
| "EMAIL_FROM_ADDRESS"
@@ -18,21 +21,24 @@ export type SendBlobStorageExportFailedEmailParams = {
projectName: string;
settingsUrl: string;
receiverEmails: string[];
- // When true, the export was disabled after repeated failures (needs the
- // customer to fix config and re-enable) rather than a transient failure.
+ // When true, the export was disabled and needs the customer to fix their
+ // config and re-enable it, rather than a transient failure.
disabled?: boolean;
};
-export const sendBlobStorageExportFailedEmail = async ({
- env,
- projectName,
- settingsUrl,
- receiverEmails,
- disabled = false,
-}: SendBlobStorageExportFailedEmailParams) => {
+export const sendIntegrationExportFailedEmail = async (
+ copy: IntegrationExportFailedEmailCopy,
+ {
+ env,
+ projectName,
+ settingsUrl,
+ receiverEmails,
+ disabled = false,
+ }: SendIntegrationExportFailedEmailParams,
+) => {
if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
logger.error(
- "Missing environment variables for sending blob storage export failed email.",
+ `Missing environment variables for sending ${copy.inlineLabel} export failed email.`,
);
return;
}
@@ -45,10 +51,11 @@ export const sendBlobStorageExportFailedEmail = async ({
const mailer = createMailTransport(env.SMTP_CONNECTION_URL);
const safeProjectName = sanitizeEmailSubject(projectName);
const subject = disabled
- ? `Blob storage export disabled for "${safeProjectName}" – action required`
- : `Blob storage export failed for "${safeProjectName}"`;
+ ? `${copy.subjectLabel} export disabled for "${safeProjectName}" – action required`
+ : `${copy.subjectLabel} export failed for "${safeProjectName}"`;
const html = await render(
- BlobStorageExportFailedEmailTemplate({
+ IntegrationExportFailedEmailTemplate({
+ copy,
projectName: safeProjectName,
settingsUrl,
disabled,
@@ -81,6 +88,9 @@ export const sendBlobStorageExportFailedEmail = async ({
await mailer.sendMail(mailOptions);
} catch (error) {
- logger.error("Failed to send blob storage export failed email", error);
+ logger.error(
+ `Failed to send ${copy.inlineLabel} export failed email`,
+ error,
+ );
}
};
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index 2cca794ec9..8f563e1ce7 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -1,7 +1,7 @@
import { pipeline, Transform, type Readable } from "stream";
import { monitorEventLoopDelay } from "perf_hooks";
import { Job } from "bullmq";
-import { prisma, Prisma } from "@langfuse/shared/src/db";
+import { prisma } from "@langfuse/shared/src/db";
import {
QueueName,
TQueueJobTypes,
@@ -50,6 +50,7 @@ import {
BLOB_INTEGRATION_DISABLED_METRIC,
classifyCustomerFault,
} from "./isCustomerFaultError";
+import { isRecordNotFoundError } from "../integrations/prismaErrors";
import { ByteCounter, TimedByteCounter } from "./byteCounters";
import { WORKER_HOST_ID } from "../../utils/hostId";
import {
@@ -1158,13 +1159,6 @@ const removeBlobExportDeprecationNotice = async (params: {
}
};
-// LFE-14894: the integration can be deleted while a run is in flight; a row
-// write racing that delete throws P2025, which must mark the job obsolete
-// rather than fail it.
-const isRecordNotFoundError = (error: unknown): boolean =>
- error instanceof Prisma.PrismaClientKnownRequestError &&
- error.code === "P2025";
-
export const handleBlobStorageIntegrationProjectJob = async (
job: Job<TQueueJobTypes[QueueName.BlobStorageIntegrationProcessingQueue]>,
) => {
diff --git a/worker/src/features/blobstorage/isCustomerFaultError.test.ts b/worker/src/features/blobstorage/isCustomerFaultError.test.ts
index d9303495e6..dae5690c0b 100644
--- a/worker/src/features/blobstorage/isCustomerFaultError.test.ts
+++ b/worker/src/features/blobstorage/isCustomerFaultError.test.ts
@@ -5,288 +5,28 @@ import {
isCustomerFaultError,
} from "./isCustomerFaultError";
-// Builds the wrapper the handler actually sees: StorageService.handleStorageError
-// rethrows SDK errors as `new Error("Failed to ...", { cause: sdkError })`.
-const wrapped = (sdkError: unknown): Error =>
- new Error("Failed to upload file to S3", { cause: sdkError });
-
-// Minimal AWS SDK v3 S3ServiceException shape: `.name`/`.Code` carry the error
-// code, `$metadata.httpStatusCode` carries the HTTP status.
-const s3Error = (name: string, httpStatusCode: number): Error => {
- const err = new Error(`${name} error`);
- err.name = name;
- Object.assign(err, { Code: name, $metadata: { httpStatusCode } });
- return err;
-};
-
-// Minimal Azure @azure/storage-blob RestError shape: `.code` + `.statusCode`.
-const azureError = (code: string, statusCode: number): Error => {
- const err = new Error(`${code} error`);
- err.name = "RestError";
- Object.assign(err, { code, statusCode });
- return err;
-};
-
-// Minimal GCS JSON-API error shape.
-const gcsError = (
- code: number,
- reason: string,
-): Error & { code: number; errors: { reason: string }[] } => {
- const err = new Error(`GCS ${reason}`) as Error & {
- code: number;
- errors: { reason: string }[];
- };
- Object.assign(err, { code, errors: [{ reason }] });
- return err;
-};
-
-describe("isCustomerFaultError", () => {
- describe("customer_fault β€” S3 / S3-compatible", () => {
- it.each([
- ["InvalidAccessKeyId", 403],
- ["SignatureDoesNotMatch", 403],
- ["AccessDenied", 403],
- ["AllAccessDisabled", 403],
- ["AccountProblem", 403],
- ["AuthorizationHeaderMalformed", 400],
- ["NoSuchBucket", 404],
- ["InvalidBucketName", 400],
- ])("classifies %s as customer_fault", (code, status) => {
- expect(isCustomerFaultError(wrapped(s3Error(code, status)))).toBe(true);
- });
-
- it("classifies a raw (unwrapped) S3 error too", () => {
- expect(isCustomerFaultError(s3Error("AccessDenied", 403))).toBe(true);
- });
- });
-
- describe("customer_fault β€” Azure", () => {
- it.each([
- ["AuthenticationFailed", 403],
- ["AuthorizationFailure", 403],
- ["AuthorizationPermissionMismatch", 403],
- ["InvalidAuthenticationInfo", 400],
- ["AccountIsDisabled", 403],
- ["InsufficientAccountPermissions", 403],
- ["ContainerNotFound", 404],
- ["InvalidResourceName", 400],
- ])("classifies %s as customer_fault", (code, status) => {
- expect(isCustomerFaultError(wrapped(azureError(code, status)))).toBe(
- true,
- );
- });
- });
-
- describe("customer_fault β€” GCS JSON API", () => {
- it("classifies 403 forbidden as customer_fault", () => {
- expect(isCustomerFaultError(wrapped(gcsError(403, "forbidden")))).toBe(
- true,
- );
- });
- });
-
- describe("customer_fault β€” bare HTTP status", () => {
- it("classifies a 401 with no recognized code as customer_fault", () => {
- const err = new Error("Unauthorized");
- Object.assign(err, { $metadata: { httpStatusCode: 401 } });
- expect(isCustomerFaultError(wrapped(err))).toBe(true);
- });
- });
-
- describe("other β€” transient / infra / unknown (bias toward investigation)", () => {
- it.each(["ETIMEDOUT", "ECONNRESET", "EAI_AGAIN", "ECONNREFUSED", "EPIPE"])(
- "classifies network error %s as other",
- (code) => {
- const err = new Error(`network ${code}`);
- Object.assign(err, { code });
- expect(isCustomerFaultError(wrapped(err))).toBe(false);
- },
- );
-
- it("classifies a 503 ServiceUnavailable as other", () => {
- expect(
- isCustomerFaultError(wrapped(s3Error("ServiceUnavailable", 503))),
- ).toBe(false);
- });
-
- it("classifies a 500 InternalError as other", () => {
- expect(isCustomerFaultError(wrapped(s3Error("InternalError", 500)))).toBe(
- false,
- );
- });
-
- it("classifies S3 SlowDown / throttling as other", () => {
- expect(isCustomerFaultError(wrapped(s3Error("SlowDown", 503)))).toBe(
- false,
- );
- });
-
- it("classifies a bare 403 with no recognized code as other (e.g. clock skew)", () => {
- expect(
- isCustomerFaultError(wrapped(s3Error("RequestTimeTooSkewed", 403))),
- ).toBe(false);
- });
-
- it("classifies a RequestTimeout as other", () => {
- expect(
- isCustomerFaultError(wrapped(s3Error("RequestTimeout", 400))),
- ).toBe(false);
- });
-
- it("classifies a plain application error as other", () => {
- expect(
- isCustomerFaultError(
- new Error("The configured export source includes enriched ..."),
- ),
- ).toBe(false);
- });
-
- it("classifies a GCS object-level notFound as other", () => {
- expect(isCustomerFaultError(wrapped(gcsError(404, "notFound")))).toBe(
- false,
- );
- });
-
- it("classifies null/undefined/non-error inputs as other", () => {
- expect(isCustomerFaultError(null)).toBe(false);
- expect(isCustomerFaultError(undefined)).toBe(false);
- expect(isCustomerFaultError("AccessDenied")).toBe(false);
- expect(isCustomerFaultError(403)).toBe(false);
- });
- });
-
- describe("customer_fault β€” outbound-URL / SSRF validation rejection", () => {
- it.each([
+// Forwarding smoke test only. This module re-exports the classifier from
+// ../integrations/customerFaultClassification, which owns the exhaustive case
+// matrix (provider codes, HTTP-status rules, cause-chain traversal). Importing
+// from THIS path on purpose: the point is to catch a broken re-export surface,
+// which typechecking alone would not β€” a re-export can satisfy its types and
+// still resolve to the wrong binding at runtime.
+describe("blob storage customer-fault re-export", () => {
+ it("forwards a customer fault to the shared classifier", () => {
+ const err = new OutboundUrlValidationError(
"blocked-ip",
- "blocked-hostname",
- "invalid-syntax",
- "invalid-encoding",
- "https-required",
- "protocol-not-allowed",
- "url-credentials-not-allowed",
- ] as const)(
- "classifies a %s validation rejection as customer_fault",
- (code) => {
- const err = new OutboundUrlValidationError(code, "blocked");
- expect(isCustomerFaultError(err)).toBe(true);
- expect(isCustomerFaultError(wrapped(err))).toBe(true);
- },
- );
-
- it("classifies dns-lookup-failed as other, not customer_fault", () => {
- // Resolvability depends on runtime resolver state, not the endpoint
- // config: a transient DNS outage across the retry window must not
- // permanently disable a working integration. Keep it investigable,
- // consistent with the network EAI_AGAIN case above.
- const err = new OutboundUrlValidationError(
- "dns-lookup-failed",
- "DNS lookup failed for host.example",
- );
- expect(isCustomerFaultError(err)).toBe(false);
- expect(isCustomerFaultError(wrapped(err))).toBe(false);
- });
-
- it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {
- // Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying
- // the same deterministic code, with no `cause` chained.
- const rewrapped = new OutboundUrlValidationError(
- "blocked-ip",
- "Blocked IP address detected For self-hosted deployments ...",
- );
- expect(isCustomerFaultError(wrapped(rewrapped))).toBe(true);
- });
-
- it("does NOT classify a code-less error that merely shares the message as customer_fault", () => {
- // Classification is code-based, not message-based: a bare 'DNS lookup
- // failed' Error (e.g. some future unrelated throw site) stays investigable.
- const err = new Error("DNS lookup failed for host.example");
- expect(isCustomerFaultError(wrapped(err))).toBe(false);
- });
- });
-
- describe("cause-chain traversal", () => {
- it("finds a customer fault nested several causes deep", () => {
- const root = s3Error("InvalidAccessKeyId", 403);
- const mid = new Error("mid", { cause: root });
- const top = new Error("top", { cause: mid });
- expect(isCustomerFaultError(top)).toBe(true);
- });
-
- it("does not infinite-loop on a cyclic cause chain", () => {
- const a = new Error("a");
- const b = new Error("b", { cause: a });
- Object.assign(a, { cause: b });
- expect(isCustomerFaultError(a)).toBe(false);
- });
- });
-});
-
-describe("classifyCustomerFault β€” disable reason buckets", () => {
- it.each([
- ["blocked-ip", "ssrf_blocked_endpoint"],
- ["blocked-hostname", "ssrf_blocked_endpoint"],
- ["invalid-syntax", "invalid_endpoint_url"],
- ["invalid-encoding", "invalid_endpoint_url"],
- ["https-required", "invalid_endpoint_url"],
- ["protocol-not-allowed", "invalid_endpoint_url"],
- ["url-credentials-not-allowed", "invalid_endpoint_url"],
- ] as const)("maps outbound-url %s to %s", (code, reason) => {
- const err = new OutboundUrlValidationError(code, "blocked");
- expect(classifyCustomerFault(err)).toBe(reason);
- expect(classifyCustomerFault(wrapped(err))).toBe(reason);
- });
-
- it.each(["InvalidAccessKeyId", "AccessDenied", "SignatureDoesNotMatch"])(
- "maps S3 auth code %s to credentials",
- (code) => {
- expect(classifyCustomerFault(wrapped(s3Error(code, 403)))).toBe(
- "credentials",
- );
- },
- );
-
- it.each(["AuthenticationFailed", "InsufficientAccountPermissions"])(
- "maps Azure auth code %s to credentials",
- (code) => {
- expect(classifyCustomerFault(wrapped(azureError(code, 403)))).toBe(
- "credentials",
- );
- },
- );
-
- it("maps a bare 401 and GCS forbidden to credentials", () => {
- const unauthorized = new Error("Unauthorized");
- Object.assign(unauthorized, { $metadata: { httpStatusCode: 401 } });
- expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");
- expect(classifyCustomerFault(wrapped(gcsError(403, "forbidden")))).toBe(
- "credentials",
+ "Blocked IP address detected",
);
+ expect(isCustomerFaultError(err)).toBe(true);
+ expect(classifyCustomerFault(err)).toBe("ssrf_blocked_endpoint");
});
- it.each([
- ["NoSuchBucket", 404],
- ["InvalidBucketName", 400],
- ["ContainerNotFound", 404],
- ["InvalidResourceName", 400],
- ] as const)("maps %s to bucket_or_container", (code, status) => {
- // extractErrorCodes reads .name/.Code/.code, so the code bucket is
- // provider-agnostic here β€” the s3Error shape is enough to exercise it.
- expect(classifyCustomerFault(wrapped(s3Error(code, status)))).toBe(
- "bucket_or_container",
+ it("forwards a non-disabling fault to the shared classifier", () => {
+ const err = new OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed for host.example",
);
- });
-
- it("returns undefined for non-disabling errors (dns-lookup-failed, transient)", () => {
- expect(
- classifyCustomerFault(
- new OutboundUrlValidationError(
- "dns-lookup-failed",
- "DNS lookup failed",
- ),
- ),
- ).toBeUndefined();
- const transient = new Error("network EAI_AGAIN");
- Object.assign(transient, { code: "EAI_AGAIN" });
- expect(classifyCustomerFault(wrapped(transient))).toBeUndefined();
+ expect(isCustomerFaultError(err)).toBe(false);
+ expect(classifyCustomerFault(err)).toBeUndefined();
});
});
diff --git a/worker/src/features/blobstorage/isCustomerFaultError.ts b/worker/src/features/blobstorage/isCustomerFaultError.ts
index f468f79691..90ec9eb049 100644
--- a/worker/src/features/blobstorage/isCustomerFaultError.ts
+++ b/worker/src/features/blobstorage/isCustomerFaultError.ts
@@ -1,11 +1,8 @@
-// True when a caught blob-export error is a deterministic customer-config /
-// credential fault. The handler uses this (after BullMQ exhausts retries) to
-// disable the integration.
-//
-// Conservative allowlist of stable provider error codes, biased toward false:
-// a false positive disables a working integration, a false negative only keeps
-// retrying. Errors arrive wrapped via `new Error(..., { cause })`
-// (StorageService.handleStorageError), so we walk the cause chain.
+export {
+ classifyCustomerFault,
+ isCustomerFaultError,
+ type CustomerFaultReason,
+} from "../integrations/customerFaultClassification";
// Counter for integrations auto-disabled after a terminal customer fault, tagged
// by `reason`. Lets a rollout regression (a classifier bug mass-disabling
@@ -13,170 +10,3 @@
// near-zero baseline, separately from expected SSRF/abuse disables.
export const BLOB_INTEGRATION_DISABLED_METRIC =
"langfuse.blobstorage.integration_disabled.count";
-
-// Coarse cause bucket for a disable, for logs/metrics. `ssrf_blocked_endpoint`
-// is the SSRF-guard rejection (endpoint resolves to a blocked host/IP) β€” the
-// "likely abuse" bucket to watch β€” while the rest are customer misconfiguration.
-export type CustomerFaultReason =
- | "ssrf_blocked_endpoint"
- | "invalid_endpoint_url"
- | "credentials"
- | "bucket_or_container";
-
-// AWS SDK v3 sets `.name`/`.Code`; Azure RestError sets `.code`. S3-compatible
-// providers (incl. GCS S3-interop) surface the S3 codes.
-const CREDENTIAL_FAULT_CODES = new Set<string>([
- // S3 β€” auth & credentials
- "InvalidAccessKeyId",
- "SignatureDoesNotMatch",
- "AccessDenied",
- "AllAccessDisabled",
- "AccountProblem",
- "AuthorizationHeaderMalformed",
- // Azure β€” auth & credentials
- "AuthenticationFailed",
- "AuthorizationFailure",
- "AuthorizationPermissionMismatch",
- "InvalidAuthenticationInfo",
- "AccountIsDisabled",
- "InsufficientAccountPermissions",
-]);
-
-const BUCKET_FAULT_CODES = new Set<string>([
- // S3 β€” bucket & path
- "NoSuchBucket",
- "InvalidBucketName",
- // Azure β€” container & path
- "ContainerNotFound",
- "InvalidResourceName",
-]);
-
-// Langfuse outbound-URL / SSRF validation rejections (OutboundUrlValidationError
-// from @langfuse/shared/.../outbound-url), split by cause. Every code here is a
-// deterministic property of the endpoint config, so it is safe to auto-disable
-// on. We deliberately omit `dns-lookup-failed`: resolvability depends on runtime
-// resolver state, not the config, so a transient DNS outage across the retry
-// window must not permanently disable a working integration. secureLlmFetch
-// makes the same distinction (dns-lookup-failed -> "endpoint-unreachable").
-const SSRF_BLOCKED_OUTBOUND_URL_CODES = new Set<string>([
- "blocked-hostname",
- "blocked-ip",
-]);
-
-const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>([
- "invalid-syntax",
- "invalid-encoding",
- "https-required",
- "protocol-not-allowed",
- "url-credentials-not-allowed",
-]);
-
-// GCS JSON-API reasons (only reachable via a GCS-native client; S3-interop uses
-// the S3 codes above). Narrow on purpose: object-level notFound stays `other`.
-const CUSTOMER_FAULT_GCS_REASONS = new Set<string>(["forbidden"]);
-
-const MAX_CAUSE_DEPTH = 10; // guard against cyclic `cause`
-
-function* errorCauseChain(error: unknown): Generator<object> {
- let current: unknown = error;
- for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
- if (!current || typeof current !== "object") return;
- yield current;
- current = (current as { cause?: unknown }).cause;
- }
-}
-
-// `.code` is overloaded by type: a number is a GCS HTTP status (here), a string
-// is an Azure error code (extractErrorCodes). Split the reads if a future
-// provider ever reports a numeric `code: 401` that isn't a credential failure.
-function extractHttpStatus(err: object): number | undefined {
- const metaStatus = (err as { $metadata?: { httpStatusCode?: unknown } })
- .$metadata?.httpStatusCode;
- if (typeof metaStatus === "number") return metaStatus;
-
- const statusCode = (err as { statusCode?: unknown }).statusCode;
- if (typeof statusCode === "number") return statusCode;
-
- const numericCode = (err as { code?: unknown }).code;
- if (typeof numericCode === "number") return numericCode;
-
- return undefined;
-}
-
-function extractErrorCodes(err: object): string[] {
- const candidates = [
- (err as { name?: unknown }).name,
- (err as { Code?: unknown }).Code,
- (err as { code?: unknown }).code,
- ];
- return candidates.filter((c): c is string => typeof c === "string");
-}
-
-function extractGcsReasons(err: object): string[] {
- const errors = (err as { errors?: unknown }).errors;
- if (!Array.isArray(errors)) return [];
- return errors
- .map((e) =>
- e &&
- typeof e === "object" &&
- typeof (e as { reason?: unknown }).reason === "string"
- ? (e as { reason: string }).reason
- : undefined,
- )
- .filter((r): r is string => typeof r === "string");
-}
-
-// Duck-type on `.name` (survives cross-package module duplication where
-// `instanceof` is unreliable) rather than importing the class, then map the
-// deterministic code to a reason; transient reasons like `dns-lookup-failed`
-// return undefined so they stay non-disabling.
-function classifyOutboundUrlFault(
- err: object,
-): CustomerFaultReason | undefined {
- if ((err as { name?: unknown }).name !== "OutboundUrlValidationError") {
- return undefined;
- }
- const code = (err as { code?: unknown }).code;
- if (typeof code !== "string") return undefined;
- if (SSRF_BLOCKED_OUTBOUND_URL_CODES.has(code)) return "ssrf_blocked_endpoint";
- if (INVALID_URL_OUTBOUND_URL_CODES.has(code)) return "invalid_endpoint_url";
- return undefined;
-}
-
-function classifyCustomerFaultLink(
- err: object,
-): CustomerFaultReason | undefined {
- const outbound = classifyOutboundUrlFault(err);
- if (outbound) return outbound;
-
- const codes = extractErrorCodes(err);
- if (codes.some((c) => CREDENTIAL_FAULT_CODES.has(c))) return "credentials";
- if (codes.some((c) => BUCKET_FAULT_CODES.has(c)))
- return "bucket_or_container";
-
- // 401 = credentials rejected: unambiguous, so it trips without a code. A bare
- // 403/404 does not (e.g. clock-skew RequestTimeTooSkewed is a 403).
- if (extractHttpStatus(err) === 401) return "credentials";
- if (extractGcsReasons(err).some((r) => CUSTOMER_FAULT_GCS_REASONS.has(r))) {
- return "credentials";
- }
- return undefined;
-}
-
-// Returns the deterministic customer-fault reason for the first matching link in
-// the cause chain, or undefined when nothing qualifies (retry-worthy / infra /
-// transient). The handler disables on any defined reason and tags the disable
-// log + metric with it.
-export function classifyCustomerFault(
- error: unknown,
-): CustomerFaultReason | undefined {
- for (const link of errorCauseChain(error)) {
- const reason = classifyCustomerFaultLink(link);
- if (reason) return reason;
- }
- return undefined;
-}
-
-export function isCustomerFaultError(error: unknown): boolean {
- return classifyCustomerFault(error) !== undefined;
-}
diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.ts
new file mode 100644
index 0000000000..6de62a4ec1
--- /dev/null
+++ b/worker/src/features/integrations/customerFaultClassification.test.ts
@@ -0,0 +1,328 @@
+import { describe, it, expect } from "vitest";
+import { OutboundUrlValidationError } from "@langfuse/shared/src/server";
+import {
+ classifyCustomerFault,
+ isCustomerFaultError,
+} from "./customerFaultClassification";
+
+// Sole owner of the customer-fault case matrix. Every integration that
+// auto-disables on a deterministic customer-config fault classifies through
+// this module, so new provider codes or reason buckets belong here and nowhere
+// else β€” the blob storage re-export carries only a forwarding smoke test.
+
+// Builds the wrapper the handlers actually see: StorageService.handleStorageError
+// and the PostHog hostname preflight both rethrow as
+// `new Error(msg, { cause: originalError })`.
+const wrapped = (sdkError: unknown): Error =>
+ new Error("Failed to upload file to S3", { cause: sdkError });
+
+// Minimal AWS SDK v3 S3ServiceException shape: `.name`/`.Code` carry the error
+// code, `$metadata.httpStatusCode` carries the HTTP status.
+const s3Error = (name: string, httpStatusCode: number): Error => {
+ const err = new Error(`${name} error`);
+ err.name = name;
+ Object.assign(err, { Code: name, $metadata: { httpStatusCode } });
+ return err;
+};
+
+// Minimal Azure @azure/storage-blob RestError shape: `.code` + `.statusCode`.
+const azureError = (code: string, statusCode: number): Error => {
+ const err = new Error(`${code} error`);
+ err.name = "RestError";
+ Object.assign(err, { code, statusCode });
+ return err;
+};
+
+// Minimal GCS JSON-API error shape.
+const gcsError = (
+ code: number,
+ reason: string,
+): Error & { code: number; errors: { reason: string }[] } => {
+ const err = new Error(`GCS ${reason}`) as Error & {
+ code: number;
+ errors: { reason: string }[];
+ };
+ Object.assign(err, { code, errors: [{ reason }] });
+ return err;
+};
+
+// The deterministic outbound-URL allowlist and its reason buckets, in one place.
+// Both the boolean and the reason-bucket suites below are driven off this table
+// so a new code cannot be added to one and forgotten in the other.
+const OUTBOUND_URL_FAULTS = [
+ ["blocked-ip", "ssrf_blocked_endpoint"],
+ ["blocked-hostname", "ssrf_blocked_endpoint"],
+ ["invalid-syntax", "invalid_endpoint_url"],
+ ["invalid-encoding", "invalid_endpoint_url"],
+ ["https-required", "invalid_endpoint_url"],
+ ["protocol-not-allowed", "invalid_endpoint_url"],
+ ["url-credentials-not-allowed", "invalid_endpoint_url"],
+] as const;
+
+describe("isCustomerFaultError", () => {
+ describe("customer_fault β€” S3 / S3-compatible", () => {
+ it.each([
+ ["InvalidAccessKeyId", 403],
+ ["SignatureDoesNotMatch", 403],
+ ["AccessDenied", 403],
+ ["AllAccessDisabled", 403],
+ ["AccountProblem", 403],
+ ["AuthorizationHeaderMalformed", 400],
+ ["NoSuchBucket", 404],
+ ["InvalidBucketName", 400],
+ ])("classifies %s as customer_fault", (code, status) => {
+ expect(isCustomerFaultError(wrapped(s3Error(code, status)))).toBe(true);
+ });
+
+ it("classifies a raw (unwrapped) S3 error too", () => {
+ expect(isCustomerFaultError(s3Error("AccessDenied", 403))).toBe(true);
+ });
+ });
+
+ describe("customer_fault β€” Azure", () => {
+ it.each([
+ ["AuthenticationFailed", 403],
+ ["AuthorizationFailure", 403],
+ ["AuthorizationPermissionMismatch", 403],
+ ["InvalidAuthenticationInfo", 400],
+ ["AccountIsDisabled", 403],
+ ["InsufficientAccountPermissions", 403],
+ ["ContainerNotFound", 404],
+ ["InvalidResourceName", 400],
+ ])("classifies %s as customer_fault", (code, status) => {
+ expect(isCustomerFaultError(wrapped(azureError(code, status)))).toBe(
+ true,
+ );
+ });
+ });
+
+ describe("customer_fault β€” GCS JSON API", () => {
+ it("classifies 403 forbidden as customer_fault", () => {
+ expect(isCustomerFaultError(wrapped(gcsError(403, "forbidden")))).toBe(
+ true,
+ );
+ });
+ });
+
+ describe("customer_fault β€” bare HTTP status", () => {
+ it("classifies a 401 with no recognized code as customer_fault", () => {
+ const err = new Error("Unauthorized");
+ Object.assign(err, { $metadata: { httpStatusCode: 401 } });
+ expect(isCustomerFaultError(wrapped(err))).toBe(true);
+ });
+ });
+
+ describe("other β€” transient / infra / unknown (bias toward investigation)", () => {
+ it.each(["ETIMEDOUT", "ECONNRESET", "EAI_AGAIN", "ECONNREFUSED", "EPIPE"])(
+ "classifies network error %s as other",
+ (code) => {
+ const err = new Error(`network ${code}`);
+ Object.assign(err, { code });
+ expect(isCustomerFaultError(wrapped(err))).toBe(false);
+ },
+ );
+
+ it("classifies a 503 ServiceUnavailable as other", () => {
+ expect(
+ isCustomerFaultError(wrapped(s3Error("ServiceUnavailable", 503))),
+ ).toBe(false);
+ });
+
+ it("classifies a 500 InternalError as other", () => {
+ expect(isCustomerFaultError(wrapped(s3Error("InternalError", 500)))).toBe(
+ false,
+ );
+ });
+
+ it("classifies S3 SlowDown / throttling as other", () => {
+ expect(isCustomerFaultError(wrapped(s3Error("SlowDown", 503)))).toBe(
+ false,
+ );
+ });
+
+ it("classifies a bare 403 with no recognized code as other (e.g. clock skew)", () => {
+ expect(
+ isCustomerFaultError(wrapped(s3Error("RequestTimeTooSkewed", 403))),
+ ).toBe(false);
+ });
+
+ it("classifies a RequestTimeout as other", () => {
+ expect(
+ isCustomerFaultError(wrapped(s3Error("RequestTimeout", 400))),
+ ).toBe(false);
+ });
+
+ it("classifies a plain application error as other", () => {
+ expect(
+ isCustomerFaultError(
+ new Error("The configured export source includes enriched ..."),
+ ),
+ ).toBe(false);
+ });
+
+ it("classifies a GCS object-level notFound as other", () => {
+ expect(isCustomerFaultError(wrapped(gcsError(404, "notFound")))).toBe(
+ false,
+ );
+ });
+
+ it("classifies null/undefined/non-error inputs as other", () => {
+ expect(isCustomerFaultError(null)).toBe(false);
+ expect(isCustomerFaultError(undefined)).toBe(false);
+ expect(isCustomerFaultError("AccessDenied")).toBe(false);
+ expect(isCustomerFaultError(403)).toBe(false);
+ });
+ });
+
+ describe("customer_fault β€” outbound-URL / SSRF validation rejection", () => {
+ it.each(OUTBOUND_URL_FAULTS.map(([code]) => code))(
+ "classifies a %s validation rejection as customer_fault",
+ (code) => {
+ const err = new OutboundUrlValidationError(code, "blocked");
+ expect(isCustomerFaultError(err)).toBe(true);
+ expect(isCustomerFaultError(wrapped(err))).toBe(true);
+ },
+ );
+
+ it("classifies dns-lookup-failed as other, not customer_fault", () => {
+ // Resolvability depends on runtime resolver state, not the endpoint
+ // config: a transient DNS outage across the retry window must not
+ // permanently disable a working integration. Keep it investigable,
+ // consistent with the network EAI_AGAIN case above.
+ const err = new OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed for host.example",
+ );
+ expect(isCustomerFaultError(err)).toBe(false);
+ expect(isCustomerFaultError(wrapped(err))).toBe(false);
+ });
+
+ it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {
+ // Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying
+ // the same deterministic code, with no `cause` chained.
+ const rewrapped = new OutboundUrlValidationError(
+ "blocked-ip",
+ "Blocked IP address detected For self-hosted deployments ...",
+ );
+ expect(isCustomerFaultError(wrapped(rewrapped))).toBe(true);
+ });
+
+ it("does NOT classify a code-less error that merely shares the message as customer_fault", () => {
+ // Classification is code-based, not message-based: a bare 'DNS lookup
+ // failed' Error (e.g. some future unrelated throw site) stays investigable.
+ const err = new Error("DNS lookup failed for host.example");
+ expect(isCustomerFaultError(wrapped(err))).toBe(false);
+ });
+ });
+
+ describe("cause-chain traversal", () => {
+ it("finds a customer fault nested several causes deep", () => {
+ const root = s3Error("InvalidAccessKeyId", 403);
+ const mid = new Error("mid", { cause: root });
+ const top = new Error("top", { cause: mid });
+ expect(isCustomerFaultError(top)).toBe(true);
+ });
+
+ it("does not infinite-loop on a cyclic cause chain", () => {
+ const a = new Error("a");
+ const b = new Error("b", { cause: a });
+ Object.assign(a, { cause: b });
+ expect(isCustomerFaultError(a)).toBe(false);
+ });
+ });
+});
+
+describe("classifyCustomerFault β€” disable reason buckets", () => {
+ it.each(OUTBOUND_URL_FAULTS)("maps outbound-url %s to %s", (code, reason) => {
+ const err = new OutboundUrlValidationError(code, "blocked");
+ expect(classifyCustomerFault(err)).toBe(reason);
+ expect(classifyCustomerFault(wrapped(err))).toBe(reason);
+ });
+
+ it.each(["InvalidAccessKeyId", "AccessDenied", "SignatureDoesNotMatch"])(
+ "maps S3 auth code %s to credentials",
+ (code) => {
+ expect(classifyCustomerFault(wrapped(s3Error(code, 403)))).toBe(
+ "credentials",
+ );
+ },
+ );
+
+ it.each(["AuthenticationFailed", "InsufficientAccountPermissions"])(
+ "maps Azure auth code %s to credentials",
+ (code) => {
+ expect(classifyCustomerFault(wrapped(azureError(code, 403)))).toBe(
+ "credentials",
+ );
+ },
+ );
+
+ it("maps a bare 401 and GCS forbidden to credentials", () => {
+ const unauthorized = new Error("Unauthorized");
+ Object.assign(unauthorized, { $metadata: { httpStatusCode: 401 } });
+ expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");
+ expect(classifyCustomerFault(wrapped(gcsError(403, "forbidden")))).toBe(
+ "credentials",
+ );
+ });
+
+ it.each([
+ ["NoSuchBucket", 404],
+ ["InvalidBucketName", 400],
+ ["ContainerNotFound", 404],
+ ["InvalidResourceName", 400],
+ ] as const)("maps %s to bucket_or_container", (code, status) => {
+ // extractErrorCodes reads .name/.Code/.code, so the code bucket is
+ // provider-agnostic here β€” the s3Error shape is enough to exercise it.
+ expect(classifyCustomerFault(wrapped(s3Error(code, status)))).toBe(
+ "bucket_or_container",
+ );
+ });
+
+ it("returns undefined for non-disabling errors (dns-lookup-failed, transient)", () => {
+ expect(
+ classifyCustomerFault(
+ new OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed",
+ ),
+ ),
+ ).toBeUndefined();
+ const transient = new Error("network EAI_AGAIN");
+ Object.assign(transient, { code: "EAI_AGAIN" });
+ expect(classifyCustomerFault(wrapped(transient))).toBeUndefined();
+ });
+
+ // The reason bucket doubles as the disable decision (defined => disable), so
+ // every non-disabling input must yield undefined and not merely a falsy
+ // `isCustomerFaultError`. Covers the wrapped and bare-input paths that the
+ // boolean suite above asserts only through isCustomerFaultError.
+ describe("undefined for every non-disabling input", () => {
+ it("returns undefined for a wrapped dns-lookup-failed", () => {
+ const err = new OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed for host.example",
+ );
+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();
+ });
+
+ it("returns undefined for a generic infra error, raw and wrapped", () => {
+ const err = new Error("ClickHouse read failed");
+ expect(classifyCustomerFault(err)).toBeUndefined();
+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();
+ });
+
+ it("returns undefined for a code-less error sharing a fault message", () => {
+ const err = new Error("Blocked hostname detected");
+ expect(classifyCustomerFault(err)).toBeUndefined();
+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();
+ });
+
+ it("returns undefined for null, undefined, and non-error inputs", () => {
+ expect(classifyCustomerFault(null)).toBeUndefined();
+ expect(classifyCustomerFault(undefined)).toBeUndefined();
+ expect(classifyCustomerFault("blocked-hostname")).toBeUndefined();
+ expect(classifyCustomerFault(403)).toBeUndefined();
+ });
+ });
+});
diff --git a/worker/src/features/integrations/customerFaultClassification.ts b/worker/src/features/integrations/customerFaultClassification.ts
new file mode 100644
index 0000000000..e24f79c85b
--- /dev/null
+++ b/worker/src/features/integrations/customerFaultClassification.ts
@@ -0,0 +1,177 @@
+// Classifies a caught integration-export error as a deterministic
+// customer-config / credential fault. Integration handlers use this to decide
+// whether to auto-disable an integration instead of retrying/alerting forever.
+// Each feature owns its own disable metric; only the classification is shared
+// here.
+//
+// Conservative allowlist of stable provider error codes, biased toward false:
+// a false positive disables a working integration, a false negative only keeps
+// retrying. Errors arrive wrapped via `new Error(..., { cause })` (e.g.
+// StorageService.handleStorageError), so we walk the cause chain.
+
+// Coarse cause bucket for a disable, for logs/metrics. `ssrf_blocked_endpoint`
+// is the SSRF-guard rejection (endpoint resolves to a blocked host/IP) β€” the
+// "likely abuse" bucket to watch β€” while the rest are customer misconfiguration.
+export type CustomerFaultReason =
+ | "ssrf_blocked_endpoint"
+ | "invalid_endpoint_url"
+ | "credentials"
+ | "bucket_or_container";
+
+// AWS SDK v3 sets `.name`/`.Code`; Azure RestError sets `.code`. S3-compatible
+// providers (incl. GCS S3-interop) surface the S3 codes.
+const CREDENTIAL_FAULT_CODES = new Set<string>([
+ // S3 β€” auth & credentials
+ "InvalidAccessKeyId",
+ "SignatureDoesNotMatch",
+ "AccessDenied",
+ "AllAccessDisabled",
+ "AccountProblem",
+ "AuthorizationHeaderMalformed",
+ // Azure β€” auth & credentials
+ "AuthenticationFailed",
+ "AuthorizationFailure",
+ "AuthorizationPermissionMismatch",
+ "InvalidAuthenticationInfo",
+ "AccountIsDisabled",
+ "InsufficientAccountPermissions",
+]);
+
+const BUCKET_FAULT_CODES = new Set<string>([
+ // S3 β€” bucket & path
+ "NoSuchBucket",
+ "InvalidBucketName",
+ // Azure β€” container & path
+ "ContainerNotFound",
+ "InvalidResourceName",
+]);
+
+// Langfuse outbound-URL / SSRF validation rejections (OutboundUrlValidationError
+// from @langfuse/shared/.../outbound-url), split by cause. Every code here is a
+// deterministic property of the endpoint config, so it is safe to auto-disable
+// on. We deliberately omit `dns-lookup-failed`: resolvability depends on runtime
+// resolver state, not the config, so a transient DNS outage across the retry
+// window must not permanently disable a working integration. secureLlmFetch
+// makes the same distinction (dns-lookup-failed -> "endpoint-unreachable").
+const SSRF_BLOCKED_OUTBOUND_URL_CODES = new Set<string>([
+ "blocked-hostname",
+ "blocked-ip",
+]);
+
+const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>([
+ "invalid-syntax",
+ "invalid-encoding",
+ "https-required",
+ "protocol-not-allowed",
+ "url-credentials-not-allowed",
+]);
+
+// GCS JSON-API reasons (only reachable via a GCS-native client; S3-interop uses
+// the S3 codes above). Narrow on purpose: object-level notFound stays `other`.
+const CUSTOMER_FAULT_GCS_REASONS = new Set<string>(["forbidden"]);
+
+const MAX_CAUSE_DEPTH = 10; // guard against cyclic `cause`
+
+function* errorCauseChain(error: unknown): Generator<object> {
+ let current: unknown = error;
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
+ if (!current || typeof current !== "object") return;
+ yield current;
+ current = (current as { cause?: unknown }).cause;
+ }
+}
+
+// `.code` is overloaded by type: a number is a GCS HTTP status (here), a string
+// is an Azure error code (extractErrorCodes). Split the reads if a future
+// provider ever reports a numeric `code: 401` that isn't a credential failure.
+function extractHttpStatus(err: object): number | undefined {
+ const metaStatus = (err as { $metadata?: { httpStatusCode?: unknown } })
+ .$metadata?.httpStatusCode;
+ if (typeof metaStatus === "number") return metaStatus;
+
+ const statusCode = (err as { statusCode?: unknown }).statusCode;
+ if (typeof statusCode === "number") return statusCode;
+
+ const numericCode = (err as { code?: unknown }).code;
+ if (typeof numericCode === "number") return numericCode;
+
+ return undefined;
+}
+
+function extractErrorCodes(err: object): string[] {
+ const candidates = [
+ (err as { name?: unknown }).name,
+ (err as { Code?: unknown }).Code,
+ (err as { code?: unknown }).code,
+ ];
+ return candidates.filter((c): c is string => typeof c === "string");
+}
+
+function extractGcsReasons(err: object): string[] {
+ const errors = (err as { errors?: unknown }).errors;
+ if (!Array.isArray(errors)) return [];
+ return errors
+ .map((e) =>
+ e &&
+ typeof e === "object" &&
+ typeof (e as { reason?: unknown }).reason === "string"
+ ? (e as { reason: string }).reason
+ : undefined,
+ )
+ .filter((r): r is string => typeof r === "string");
+}
+
+// Duck-type on `.name` (survives cross-package module duplication where
+// `instanceof` is unreliable) rather than importing the class, then map the
+// deterministic code to a reason; transient reasons like `dns-lookup-failed`
+// return undefined so they stay non-disabling.
+function classifyOutboundUrlFault(
+ err: object,
+): CustomerFaultReason | undefined {
+ if ((err as { name?: unknown }).name !== "OutboundUrlValidationError") {
+ return undefined;
+ }
+ const code = (err as { code?: unknown }).code;
+ if (typeof code !== "string") return undefined;
+ if (SSRF_BLOCKED_OUTBOUND_URL_CODES.has(code)) return "ssrf_blocked_endpoint";
+ if (INVALID_URL_OUTBOUND_URL_CODES.has(code)) return "invalid_endpoint_url";
+ return undefined;
+}
+
+function classifyCustomerFaultLink(
+ err: object,
+): CustomerFaultReason | undefined {
+ const outbound = classifyOutboundUrlFault(err);
+ if (outbound) return outbound;
+
+ const codes = extractErrorCodes(err);
+ if (codes.some((c) => CREDENTIAL_FAULT_CODES.has(c))) return "credentials";
+ if (codes.some((c) => BUCKET_FAULT_CODES.has(c)))
+ return "bucket_or_container";
+
+ // 401 = credentials rejected: unambiguous, so it trips without a code. A bare
+ // 403/404 does not (e.g. clock-skew RequestTimeTooSkewed is a 403).
+ if (extractHttpStatus(err) === 401) return "credentials";
+ if (extractGcsReasons(err).some((r) => CUSTOMER_FAULT_GCS_REASONS.has(r))) {
+ return "credentials";
+ }
+ return undefined;
+}
+
+// Returns the deterministic customer-fault reason for the first matching link in
+// the cause chain, or undefined when nothing qualifies (retry-worthy / infra /
+// transient). The handler disables on any defined reason and tags the disable
+// log + metric with it.
+export function classifyCustomerFault(
+ error: unknown,
+): CustomerFaultReason | undefined {
+ for (const link of errorCauseChain(error)) {
+ const reason = classifyCustomerFaultLink(link);
+ if (reason) return reason;
+ }
+ return undefined;
+}
+
+export function isCustomerFaultError(error: unknown): boolean {
+ return classifyCustomerFault(error) !== undefined;
+}
diff --git a/worker/src/features/integrations/prismaErrors.ts b/worker/src/features/integrations/prismaErrors.ts
new file mode 100644
index 0000000000..032c35b2dc
--- /dev/null
+++ b/worker/src/features/integrations/prismaErrors.ts
@@ -0,0 +1,8 @@
+import { Prisma } from "@langfuse/shared/src/db";
+
+// An integration can be deleted while one of its runs is still in flight; a row
+// write racing that delete throws P2025. Integration handlers treat that as
+// "job is obsolete, complete it quietly" rather than as a failure to retry.
+export const isRecordNotFoundError = (error: unknown): boolean =>
+ error instanceof Prisma.PrismaClientKnownRequestError &&
+ error.code === "P2025";