πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(worker,shared): share integration helpers across export integrations (#16094)
Niklas Semmler7c84312f69fa
diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsxnew file mode 100644index 0000000000..4ddc49de72--- /dev/null+++ b/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsx@@ -0,0 +1,29 @@+import React from "react";+import {+ sendIntegrationExportFailedEmail,+ type SendIntegrationExportFailedEmailParams,+} from "../integrationExportFailed/sendIntegrationExportFailedEmail";+import { type IntegrationExportFailedEmailCopy } from "../integrationExportFailed/IntegrationExportFailedEmailTemplate";++export type SendBlobStorageExportFailedEmailParams =+ SendIntegrationExportFailedEmailParams;++export const blobStorageExportFailedEmailCopy: IntegrationExportFailedEmailCopy =+ {+ subjectLabel: "Blob storage",+ headingLabel: "Blob Storage",+ inlineLabel: "blob storage",+ disabledBody: (projectName) => (+ <>+ The blob storage export for project "{projectName}+ " has been disabled after repeated failures. This usually means the+ destination configuration or credentials are no longer valid. Once you+ have updated them, simply re-enable the export in the integration+ settings and it will pick up right where it left off.+ </>+ ),+ };++export const sendBlobStorageExportFailedEmail = (+ params: SendBlobStorageExportFailedEmailParams,+) => sendIntegrationExportFailedEmail(blobStorageExportFailedEmailCopy, params);diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsx b/packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsxsimilarity index 64%rename from packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsxrename to packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsxindex 4efffae4e9..784a11bfa7 100644--- a/packages/shared/src/server/services/email/blobStorageExportFailed/BlobStorageExportFailedEmailTemplate.tsx+++ b/packages/shared/src/server/services/email/integrationExportFailed/IntegrationExportFailedEmailTemplate.tsx@@ -1,4 +1,4 @@-import React from "react";+import React, { type ReactNode } from "react";import {Body,Button,@@ -14,22 +14,33 @@ import {Text,} from "@react-email/components";-type BlobStorageExportFailedEmailTemplateProps = {+export type IntegrationExportFailedEmailCopy = {+ /** Sentence-initial, for the subject and preview: "Blob storage export failed for β¦". */+ subjectLabel: string;+ /** Title Case, for the email heading: "Blob Storage Export Disabled". */+ headingLabel: string;+ /** Mid-sentence, for body prose and log lines: "The scheduled blob storage export β¦". */+ inlineLabel: string;+ /** Shown when the export was turned off; the remediation is integration specific. */+ disabledBody: (projectName: string) => ReactNode;+};++type IntegrationExportFailedEmailTemplateProps = {+ copy: IntegrationExportFailedEmailCopy;projectName: string;settingsUrl: string;- // When true, the export was turned off after repeated failures and the- // customer must fix their config and re-enable it β not a transient failure.disabled?: boolean;};-export const BlobStorageExportFailedEmailTemplate = ({+export const IntegrationExportFailedEmailTemplate = ({+ copy,projectName,settingsUrl,disabled = false,-}: BlobStorageExportFailedEmailTemplateProps) => {+}: IntegrationExportFailedEmailTemplateProps) => {const preview = disabled- ? `Blob storage export disabled for project "${projectName}"`- : `Blob storage export failed for project "${projectName}"`;+ ? `${copy.subjectLabel} export disabled for project "${projectName}"`+ : `${copy.subjectLabel} export failed for project "${projectName}"`;return (<Html><Head />@@ -50,22 +61,15 @@ export const BlobStorageExportFailedEmailTemplate = ({<Section><Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">{disabled- ? "Blob Storage Export Disabled"- : "Blob Storage Export Failed"}+ ? `${copy.headingLabel} Export Disabled`+ : `${copy.headingLabel} Export Failed`}</Heading><Text className="text-gray-700 text-sm leading-6">{disabled ? (- <>- The blob storage export for project "{projectName}- " has been disabled after repeated failures. This- usually means the destination configuration or credentials- are no longer valid. Once you have updated them, simply- re-enable the export in the integration settings and it will- pick up right where it left off.- </>+ copy.disabledBody(projectName)) : (<>- The scheduled blob storage export for project "+ The scheduled {copy.inlineLabel} export for project "{projectName}" has failed after multiple attempts. Itwill be retried automatically at the next scheduled export.If the issue persists, review the integration settings to@@ -89,7 +93,7 @@ export const BlobStorageExportFailedEmailTemplate = ({<Section><Text className="text-[#666666] text-[12px] leading-[24px]">This notification was sent to project admins regarding a{" "}- {disabled ? "disabled" : "failed"} blob storage export for+ {disabled ? "disabled" : "failed"} {copy.inlineLabel} export forproject "{projectName}".</Text></Section>diff --git a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.ts b/packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.tssimilarity index 62%rename from packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.tsrename to packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.tsindex 06eef9d958..1aceb47de5 100644--- a/packages/shared/src/server/services/email/blobStorageExportFailed/sendBlobStorageExportFailedEmail.ts+++ b/packages/shared/src/server/services/email/integrationExportFailed/sendIntegrationExportFailedEmail.ts@@ -3,9 +3,12 @@ import { createMailTransport } from "../transport";import { z } from "zod";import { sanitizeEmailSubject } from "../../../../utils/zod";import { logger } from "../../../logger";-import { BlobStorageExportFailedEmailTemplate } from "./BlobStorageExportFailedEmailTemplate";+import {+ IntegrationExportFailedEmailTemplate,+ type IntegrationExportFailedEmailCopy,+} from "./IntegrationExportFailedEmailTemplate";-export type SendBlobStorageExportFailedEmailParams = {+export type SendIntegrationExportFailedEmailParams = {env: Partial<Record<| "EMAIL_FROM_ADDRESS"@@ -18,21 +21,24 @@ export type SendBlobStorageExportFailedEmailParams = {projectName: string;settingsUrl: string;receiverEmails: string[];- // When true, the export was disabled after repeated failures (needs the- // customer to fix config and re-enable) rather than a transient failure.+ // When true, the export was disabled and needs the customer to fix their+ // config and re-enable it, rather than a transient failure.disabled?: boolean;};-export const sendBlobStorageExportFailedEmail = async ({- env,- projectName,- settingsUrl,- receiverEmails,- disabled = false,-}: SendBlobStorageExportFailedEmailParams) => {+export const sendIntegrationExportFailedEmail = async (+ copy: IntegrationExportFailedEmailCopy,+ {+ env,+ projectName,+ settingsUrl,+ receiverEmails,+ disabled = false,+ }: SendIntegrationExportFailedEmailParams,+) => {if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {logger.error(- "Missing environment variables for sending blob storage export failed email.",+ `Missing environment variables for sending ${copy.inlineLabel} export failed email.`,);return;}@@ -45,10 +51,11 @@ export const sendBlobStorageExportFailedEmail = async ({const mailer = createMailTransport(env.SMTP_CONNECTION_URL);const safeProjectName = sanitizeEmailSubject(projectName);const subject = disabled- ? `Blob storage export disabled for "${safeProjectName}" β action required`- : `Blob storage export failed for "${safeProjectName}"`;+ ? `${copy.subjectLabel} export disabled for "${safeProjectName}" β action required`+ : `${copy.subjectLabel} export failed for "${safeProjectName}"`;const html = await render(- BlobStorageExportFailedEmailTemplate({+ IntegrationExportFailedEmailTemplate({+ copy,projectName: safeProjectName,settingsUrl,disabled,@@ -81,6 +88,9 @@ export const sendBlobStorageExportFailedEmail = async ({await mailer.sendMail(mailOptions);} catch (error) {- logger.error("Failed to send blob storage export failed email", error);+ logger.error(+ `Failed to send ${copy.inlineLabel} export failed email`,+ error,+ );}};diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex 2cca794ec9..8f563e1ce7 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -1,7 +1,7 @@import { pipeline, Transform, type Readable } from "stream";import { monitorEventLoopDelay } from "perf_hooks";import { Job } from "bullmq";-import { prisma, Prisma } from "@langfuse/shared/src/db";+import { prisma } from "@langfuse/shared/src/db";import {QueueName,TQueueJobTypes,@@ -50,6 +50,7 @@ import {BLOB_INTEGRATION_DISABLED_METRIC,classifyCustomerFault,} from "./isCustomerFaultError";+import { isRecordNotFoundError } from "../integrations/prismaErrors";import { ByteCounter, TimedByteCounter } from "./byteCounters";import { WORKER_HOST_ID } from "../../utils/hostId";import {@@ -1158,13 +1159,6 @@ const removeBlobExportDeprecationNotice = async (params: {}};-// LFE-14894: the integration can be deleted while a run is in flight; a row-// write racing that delete throws P2025, which must mark the job obsolete-// rather than fail it.-const isRecordNotFoundError = (error: unknown): boolean =>- error instanceof Prisma.PrismaClientKnownRequestError &&- error.code === "P2025";-export const handleBlobStorageIntegrationProjectJob = async (job: Job<TQueueJobTypes[QueueName.BlobStorageIntegrationProcessingQueue]>,) => {diff --git a/worker/src/features/blobstorage/isCustomerFaultError.test.ts b/worker/src/features/blobstorage/isCustomerFaultError.test.tsindex d9303495e6..dae5690c0b 100644--- a/worker/src/features/blobstorage/isCustomerFaultError.test.ts+++ b/worker/src/features/blobstorage/isCustomerFaultError.test.ts@@ -5,288 +5,28 @@ import {isCustomerFaultError,} from "./isCustomerFaultError";-// Builds the wrapper the handler actually sees: StorageService.handleStorageError-// rethrows SDK errors as `new Error("Failed to ...", { cause: sdkError })`.-const wrapped = (sdkError: unknown): Error =>- new Error("Failed to upload file to S3", { cause: sdkError });--// Minimal AWS SDK v3 S3ServiceException shape: `.name`/`.Code` carry the error-// code, `$metadata.httpStatusCode` carries the HTTP status.-const s3Error = (name: string, httpStatusCode: number): Error => {- const err = new Error(`${name} error`);- err.name = name;- Object.assign(err, { Code: name, $metadata: { httpStatusCode } });- return err;-};--// Minimal Azure @azure/storage-blob RestError shape: `.code` + `.statusCode`.-const azureError = (code: string, statusCode: number): Error => {- const err = new Error(`${code} error`);- err.name = "RestError";- Object.assign(err, { code, statusCode });- return err;-};--// Minimal GCS JSON-API error shape.-const gcsError = (- code: number,- reason: string,-): Error & { code: number; errors: { reason: string }[] } => {- const err = new Error(`GCS ${reason}`) as Error & {- code: number;- errors: { reason: string }[];- };- Object.assign(err, { code, errors: [{ reason }] });- return err;-};--describe("isCustomerFaultError", () => {- describe("customer_fault β S3 / S3-compatible", () => {- it.each([- ["InvalidAccessKeyId", 403],- ["SignatureDoesNotMatch", 403],- ["AccessDenied", 403],- ["AllAccessDisabled", 403],- ["AccountProblem", 403],- ["AuthorizationHeaderMalformed", 400],- ["NoSuchBucket", 404],- ["InvalidBucketName", 400],- ])("classifies %s as customer_fault", (code, status) => {- expect(isCustomerFaultError(wrapped(s3Error(code, status)))).toBe(true);- });-- it("classifies a raw (unwrapped) S3 error too", () => {- expect(isCustomerFaultError(s3Error("AccessDenied", 403))).toBe(true);- });- });-- describe("customer_fault β Azure", () => {- it.each([- ["AuthenticationFailed", 403],- ["AuthorizationFailure", 403],- ["AuthorizationPermissionMismatch", 403],- ["InvalidAuthenticationInfo", 400],- ["AccountIsDisabled", 403],- ["InsufficientAccountPermissions", 403],- ["ContainerNotFound", 404],- ["InvalidResourceName", 400],- ])("classifies %s as customer_fault", (code, status) => {- expect(isCustomerFaultError(wrapped(azureError(code, status)))).toBe(- true,- );- });- });-- describe("customer_fault β GCS JSON API", () => {- it("classifies 403 forbidden as customer_fault", () => {- expect(isCustomerFaultError(wrapped(gcsError(403, "forbidden")))).toBe(- true,- );- });- });-- describe("customer_fault β bare HTTP status", () => {- it("classifies a 401 with no recognized code as customer_fault", () => {- const err = new Error("Unauthorized");- Object.assign(err, { $metadata: { httpStatusCode: 401 } });- expect(isCustomerFaultError(wrapped(err))).toBe(true);- });- });-- describe("other β transient / infra / unknown (bias toward investigation)", () => {- it.each(["ETIMEDOUT", "ECONNRESET", "EAI_AGAIN", "ECONNREFUSED", "EPIPE"])(- "classifies network error %s as other",- (code) => {- const err = new Error(`network ${code}`);- Object.assign(err, { code });- expect(isCustomerFaultError(wrapped(err))).toBe(false);- },- );-- it("classifies a 503 ServiceUnavailable as other", () => {- expect(- isCustomerFaultError(wrapped(s3Error("ServiceUnavailable", 503))),- ).toBe(false);- });-- it("classifies a 500 InternalError as other", () => {- expect(isCustomerFaultError(wrapped(s3Error("InternalError", 500)))).toBe(- false,- );- });-- it("classifies S3 SlowDown / throttling as other", () => {- expect(isCustomerFaultError(wrapped(s3Error("SlowDown", 503)))).toBe(- false,- );- });-- it("classifies a bare 403 with no recognized code as other (e.g. clock skew)", () => {- expect(- isCustomerFaultError(wrapped(s3Error("RequestTimeTooSkewed", 403))),- ).toBe(false);- });-- it("classifies a RequestTimeout as other", () => {- expect(- isCustomerFaultError(wrapped(s3Error("RequestTimeout", 400))),- ).toBe(false);- });-- it("classifies a plain application error as other", () => {- expect(- isCustomerFaultError(- new Error("The configured export source includes enriched ..."),- ),- ).toBe(false);- });-- it("classifies a GCS object-level notFound as other", () => {- expect(isCustomerFaultError(wrapped(gcsError(404, "notFound")))).toBe(- false,- );- });-- it("classifies null/undefined/non-error inputs as other", () => {- expect(isCustomerFaultError(null)).toBe(false);- expect(isCustomerFaultError(undefined)).toBe(false);- expect(isCustomerFaultError("AccessDenied")).toBe(false);- expect(isCustomerFaultError(403)).toBe(false);- });- });-- describe("customer_fault β outbound-URL / SSRF validation rejection", () => {- it.each([+// Forwarding smoke test only. This module re-exports the classifier from+// ../integrations/customerFaultClassification, which owns the exhaustive case+// matrix (provider codes, HTTP-status rules, cause-chain traversal). Importing+// from THIS path on purpose: the point is to catch a broken re-export surface,+// which typechecking alone would not β a re-export can satisfy its types and+// still resolve to the wrong binding at runtime.+describe("blob storage customer-fault re-export", () => {+ it("forwards a customer fault to the shared classifier", () => {+ const err = new OutboundUrlValidationError("blocked-ip",- "blocked-hostname",- "invalid-syntax",- "invalid-encoding",- "https-required",- "protocol-not-allowed",- "url-credentials-not-allowed",- ] as const)(- "classifies a %s validation rejection as customer_fault",- (code) => {- const err = new OutboundUrlValidationError(code, "blocked");- expect(isCustomerFaultError(err)).toBe(true);- expect(isCustomerFaultError(wrapped(err))).toBe(true);- },- );-- it("classifies dns-lookup-failed as other, not customer_fault", () => {- // Resolvability depends on runtime resolver state, not the endpoint- // config: a transient DNS outage across the retry window must not- // permanently disable a working integration. Keep it investigable,- // consistent with the network EAI_AGAIN case above.- const err = new OutboundUrlValidationError(- "dns-lookup-failed",- "DNS lookup failed for host.example",- );- expect(isCustomerFaultError(err)).toBe(false);- expect(isCustomerFaultError(wrapped(err))).toBe(false);- });-- it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {- // Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying- // the same deterministic code, with no `cause` chained.- const rewrapped = new OutboundUrlValidationError(- "blocked-ip",- "Blocked IP address detected For self-hosted deployments ...",- );- expect(isCustomerFaultError(wrapped(rewrapped))).toBe(true);- });-- it("does NOT classify a code-less error that merely shares the message as customer_fault", () => {- // Classification is code-based, not message-based: a bare 'DNS lookup- // failed' Error (e.g. some future unrelated throw site) stays investigable.- const err = new Error("DNS lookup failed for host.example");- expect(isCustomerFaultError(wrapped(err))).toBe(false);- });- });-- describe("cause-chain traversal", () => {- it("finds a customer fault nested several causes deep", () => {- const root = s3Error("InvalidAccessKeyId", 403);- const mid = new Error("mid", { cause: root });- const top = new Error("top", { cause: mid });- expect(isCustomerFaultError(top)).toBe(true);- });-- it("does not infinite-loop on a cyclic cause chain", () => {- const a = new Error("a");- const b = new Error("b", { cause: a });- Object.assign(a, { cause: b });- expect(isCustomerFaultError(a)).toBe(false);- });- });-});--describe("classifyCustomerFault β disable reason buckets", () => {- it.each([- ["blocked-ip", "ssrf_blocked_endpoint"],- ["blocked-hostname", "ssrf_blocked_endpoint"],- ["invalid-syntax", "invalid_endpoint_url"],- ["invalid-encoding", "invalid_endpoint_url"],- ["https-required", "invalid_endpoint_url"],- ["protocol-not-allowed", "invalid_endpoint_url"],- ["url-credentials-not-allowed", "invalid_endpoint_url"],- ] as const)("maps outbound-url %s to %s", (code, reason) => {- const err = new OutboundUrlValidationError(code, "blocked");- expect(classifyCustomerFault(err)).toBe(reason);- expect(classifyCustomerFault(wrapped(err))).toBe(reason);- });-- it.each(["InvalidAccessKeyId", "AccessDenied", "SignatureDoesNotMatch"])(- "maps S3 auth code %s to credentials",- (code) => {- expect(classifyCustomerFault(wrapped(s3Error(code, 403)))).toBe(- "credentials",- );- },- );-- it.each(["AuthenticationFailed", "InsufficientAccountPermissions"])(- "maps Azure auth code %s to credentials",- (code) => {- expect(classifyCustomerFault(wrapped(azureError(code, 403)))).toBe(- "credentials",- );- },- );-- it("maps a bare 401 and GCS forbidden to credentials", () => {- const unauthorized = new Error("Unauthorized");- Object.assign(unauthorized, { $metadata: { httpStatusCode: 401 } });- expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");- expect(classifyCustomerFault(wrapped(gcsError(403, "forbidden")))).toBe(- "credentials",+ "Blocked IP address detected",);+ expect(isCustomerFaultError(err)).toBe(true);+ expect(classifyCustomerFault(err)).toBe("ssrf_blocked_endpoint");});- it.each([- ["NoSuchBucket", 404],- ["InvalidBucketName", 400],- ["ContainerNotFound", 404],- ["InvalidResourceName", 400],- ] as const)("maps %s to bucket_or_container", (code, status) => {- // extractErrorCodes reads .name/.Code/.code, so the code bucket is- // provider-agnostic here β the s3Error shape is enough to exercise it.- expect(classifyCustomerFault(wrapped(s3Error(code, status)))).toBe(- "bucket_or_container",+ it("forwards a non-disabling fault to the shared classifier", () => {+ const err = new OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed for host.example",);- });-- it("returns undefined for non-disabling errors (dns-lookup-failed, transient)", () => {- expect(- classifyCustomerFault(- new OutboundUrlValidationError(- "dns-lookup-failed",- "DNS lookup failed",- ),- ),- ).toBeUndefined();- const transient = new Error("network EAI_AGAIN");- Object.assign(transient, { code: "EAI_AGAIN" });- expect(classifyCustomerFault(wrapped(transient))).toBeUndefined();+ expect(isCustomerFaultError(err)).toBe(false);+ expect(classifyCustomerFault(err)).toBeUndefined();});});diff --git a/worker/src/features/blobstorage/isCustomerFaultError.ts b/worker/src/features/blobstorage/isCustomerFaultError.tsindex f468f79691..90ec9eb049 100644--- a/worker/src/features/blobstorage/isCustomerFaultError.ts+++ b/worker/src/features/blobstorage/isCustomerFaultError.ts@@ -1,11 +1,8 @@-// True when a caught blob-export error is a deterministic customer-config /-// credential fault. The handler uses this (after BullMQ exhausts retries) to-// disable the integration.-//-// Conservative allowlist of stable provider error codes, biased toward false:-// a false positive disables a working integration, a false negative only keeps-// retrying. Errors arrive wrapped via `new Error(..., { cause })`-// (StorageService.handleStorageError), so we walk the cause chain.+export {+ classifyCustomerFault,+ isCustomerFaultError,+ type CustomerFaultReason,+} from "../integrations/customerFaultClassification";// Counter for integrations auto-disabled after a terminal customer fault, tagged// by `reason`. Lets a rollout regression (a classifier bug mass-disabling@@ -13,170 +10,3 @@// near-zero baseline, separately from expected SSRF/abuse disables.export const BLOB_INTEGRATION_DISABLED_METRIC ="langfuse.blobstorage.integration_disabled.count";--// Coarse cause bucket for a disable, for logs/metrics. `ssrf_blocked_endpoint`-// is the SSRF-guard rejection (endpoint resolves to a blocked host/IP) β the-// "likely abuse" bucket to watch β while the rest are customer misconfiguration.-export type CustomerFaultReason =- | "ssrf_blocked_endpoint"- | "invalid_endpoint_url"- | "credentials"- | "bucket_or_container";--// AWS SDK v3 sets `.name`/`.Code`; Azure RestError sets `.code`. S3-compatible-// providers (incl. GCS S3-interop) surface the S3 codes.-const CREDENTIAL_FAULT_CODES = new Set<string>([- // S3 β auth & credentials- "InvalidAccessKeyId",- "SignatureDoesNotMatch",- "AccessDenied",- "AllAccessDisabled",- "AccountProblem",- "AuthorizationHeaderMalformed",- // Azure β auth & credentials- "AuthenticationFailed",- "AuthorizationFailure",- "AuthorizationPermissionMismatch",- "InvalidAuthenticationInfo",- "AccountIsDisabled",- "InsufficientAccountPermissions",-]);--const BUCKET_FAULT_CODES = new Set<string>([- // S3 β bucket & path- "NoSuchBucket",- "InvalidBucketName",- // Azure β container & path- "ContainerNotFound",- "InvalidResourceName",-]);--// Langfuse outbound-URL / SSRF validation rejections (OutboundUrlValidationError-// from @langfuse/shared/.../outbound-url), split by cause. Every code here is a-// deterministic property of the endpoint config, so it is safe to auto-disable-// on. We deliberately omit `dns-lookup-failed`: resolvability depends on runtime-// resolver state, not the config, so a transient DNS outage across the retry-// window must not permanently disable a working integration. secureLlmFetch-// makes the same distinction (dns-lookup-failed -> "endpoint-unreachable").-const SSRF_BLOCKED_OUTBOUND_URL_CODES = new Set<string>([- "blocked-hostname",- "blocked-ip",-]);--const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>([- "invalid-syntax",- "invalid-encoding",- "https-required",- "protocol-not-allowed",- "url-credentials-not-allowed",-]);--// GCS JSON-API reasons (only reachable via a GCS-native client; S3-interop uses-// the S3 codes above). Narrow on purpose: object-level notFound stays `other`.-const CUSTOMER_FAULT_GCS_REASONS = new Set<string>(["forbidden"]);--const MAX_CAUSE_DEPTH = 10; // guard against cyclic `cause`--function* errorCauseChain(error: unknown): Generator<object> {- let current: unknown = error;- for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {- if (!current || typeof current !== "object") return;- yield current;- current = (current as { cause?: unknown }).cause;- }-}--// `.code` is overloaded by type: a number is a GCS HTTP status (here), a string-// is an Azure error code (extractErrorCodes). Split the reads if a future-// provider ever reports a numeric `code: 401` that isn't a credential failure.-function extractHttpStatus(err: object): number | undefined {- const metaStatus = (err as { $metadata?: { httpStatusCode?: unknown } })- .$metadata?.httpStatusCode;- if (typeof metaStatus === "number") return metaStatus;-- const statusCode = (err as { statusCode?: unknown }).statusCode;- if (typeof statusCode === "number") return statusCode;-- const numericCode = (err as { code?: unknown }).code;- if (typeof numericCode === "number") return numericCode;-- return undefined;-}--function extractErrorCodes(err: object): string[] {- const candidates = [- (err as { name?: unknown }).name,- (err as { Code?: unknown }).Code,- (err as { code?: unknown }).code,- ];- return candidates.filter((c): c is string => typeof c === "string");-}--function extractGcsReasons(err: object): string[] {- const errors = (err as { errors?: unknown }).errors;- if (!Array.isArray(errors)) return [];- return errors- .map((e) =>- e &&- typeof e === "object" &&- typeof (e as { reason?: unknown }).reason === "string"- ? (e as { reason: string }).reason- : undefined,- )- .filter((r): r is string => typeof r === "string");-}--// Duck-type on `.name` (survives cross-package module duplication where-// `instanceof` is unreliable) rather than importing the class, then map the-// deterministic code to a reason; transient reasons like `dns-lookup-failed`-// return undefined so they stay non-disabling.-function classifyOutboundUrlFault(- err: object,-): CustomerFaultReason | undefined {- if ((err as { name?: unknown }).name !== "OutboundUrlValidationError") {- return undefined;- }- const code = (err as { code?: unknown }).code;- if (typeof code !== "string") return undefined;- if (SSRF_BLOCKED_OUTBOUND_URL_CODES.has(code)) return "ssrf_blocked_endpoint";- if (INVALID_URL_OUTBOUND_URL_CODES.has(code)) return "invalid_endpoint_url";- return undefined;-}--function classifyCustomerFaultLink(- err: object,-): CustomerFaultReason | undefined {- const outbound = classifyOutboundUrlFault(err);- if (outbound) return outbound;-- const codes = extractErrorCodes(err);- if (codes.some((c) => CREDENTIAL_FAULT_CODES.has(c))) return "credentials";- if (codes.some((c) => BUCKET_FAULT_CODES.has(c)))- return "bucket_or_container";-- // 401 = credentials rejected: unambiguous, so it trips without a code. A bare- // 403/404 does not (e.g. clock-skew RequestTimeTooSkewed is a 403).- if (extractHttpStatus(err) === 401) return "credentials";- if (extractGcsReasons(err).some((r) => CUSTOMER_FAULT_GCS_REASONS.has(r))) {- return "credentials";- }- return undefined;-}--// Returns the deterministic customer-fault reason for the first matching link in-// the cause chain, or undefined when nothing qualifies (retry-worthy / infra /-// transient). The handler disables on any defined reason and tags the disable-// log + metric with it.-export function classifyCustomerFault(- error: unknown,-): CustomerFaultReason | undefined {- for (const link of errorCauseChain(error)) {- const reason = classifyCustomerFaultLink(link);- if (reason) return reason;- }- return undefined;-}--export function isCustomerFaultError(error: unknown): boolean {- return classifyCustomerFault(error) !== undefined;-}diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.tsnew file mode 100644index 0000000000..6de62a4ec1--- /dev/null+++ b/worker/src/features/integrations/customerFaultClassification.test.ts@@ -0,0 +1,328 @@+import { describe, it, expect } from "vitest";+import { OutboundUrlValidationError } from "@langfuse/shared/src/server";+import {+ classifyCustomerFault,+ isCustomerFaultError,+} from "./customerFaultClassification";++// Sole owner of the customer-fault case matrix. Every integration that+// auto-disables on a deterministic customer-config fault classifies through+// this module, so new provider codes or reason buckets belong here and nowhere+// else β the blob storage re-export carries only a forwarding smoke test.++// Builds the wrapper the handlers actually see: StorageService.handleStorageError+// and the PostHog hostname preflight both rethrow as+// `new Error(msg, { cause: originalError })`.+const wrapped = (sdkError: unknown): Error =>+ new Error("Failed to upload file to S3", { cause: sdkError });++// Minimal AWS SDK v3 S3ServiceException shape: `.name`/`.Code` carry the error+// code, `$metadata.httpStatusCode` carries the HTTP status.+const s3Error = (name: string, httpStatusCode: number): Error => {+ const err = new Error(`${name} error`);+ err.name = name;+ Object.assign(err, { Code: name, $metadata: { httpStatusCode } });+ return err;+};++// Minimal Azure @azure/storage-blob RestError shape: `.code` + `.statusCode`.+const azureError = (code: string, statusCode: number): Error => {+ const err = new Error(`${code} error`);+ err.name = "RestError";+ Object.assign(err, { code, statusCode });+ return err;+};++// Minimal GCS JSON-API error shape.+const gcsError = (+ code: number,+ reason: string,+): Error & { code: number; errors: { reason: string }[] } => {+ const err = new Error(`GCS ${reason}`) as Error & {+ code: number;+ errors: { reason: string }[];+ };+ Object.assign(err, { code, errors: [{ reason }] });+ return err;+};++// The deterministic outbound-URL allowlist and its reason buckets, in one place.+// Both the boolean and the reason-bucket suites below are driven off this table+// so a new code cannot be added to one and forgotten in the other.+const OUTBOUND_URL_FAULTS = [+ ["blocked-ip", "ssrf_blocked_endpoint"],+ ["blocked-hostname", "ssrf_blocked_endpoint"],+ ["invalid-syntax", "invalid_endpoint_url"],+ ["invalid-encoding", "invalid_endpoint_url"],+ ["https-required", "invalid_endpoint_url"],+ ["protocol-not-allowed", "invalid_endpoint_url"],+ ["url-credentials-not-allowed", "invalid_endpoint_url"],+] as const;++describe("isCustomerFaultError", () => {+ describe("customer_fault β S3 / S3-compatible", () => {+ it.each([+ ["InvalidAccessKeyId", 403],+ ["SignatureDoesNotMatch", 403],+ ["AccessDenied", 403],+ ["AllAccessDisabled", 403],+ ["AccountProblem", 403],+ ["AuthorizationHeaderMalformed", 400],+ ["NoSuchBucket", 404],+ ["InvalidBucketName", 400],+ ])("classifies %s as customer_fault", (code, status) => {+ expect(isCustomerFaultError(wrapped(s3Error(code, status)))).toBe(true);+ });++ it("classifies a raw (unwrapped) S3 error too", () => {+ expect(isCustomerFaultError(s3Error("AccessDenied", 403))).toBe(true);+ });+ });++ describe("customer_fault β Azure", () => {+ it.each([+ ["AuthenticationFailed", 403],+ ["AuthorizationFailure", 403],+ ["AuthorizationPermissionMismatch", 403],+ ["InvalidAuthenticationInfo", 400],+ ["AccountIsDisabled", 403],+ ["InsufficientAccountPermissions", 403],+ ["ContainerNotFound", 404],+ ["InvalidResourceName", 400],+ ])("classifies %s as customer_fault", (code, status) => {+ expect(isCustomerFaultError(wrapped(azureError(code, status)))).toBe(+ true,+ );+ });+ });++ describe("customer_fault β GCS JSON API", () => {+ it("classifies 403 forbidden as customer_fault", () => {+ expect(isCustomerFaultError(wrapped(gcsError(403, "forbidden")))).toBe(+ true,+ );+ });+ });++ describe("customer_fault β bare HTTP status", () => {+ it("classifies a 401 with no recognized code as customer_fault", () => {+ const err = new Error("Unauthorized");+ Object.assign(err, { $metadata: { httpStatusCode: 401 } });+ expect(isCustomerFaultError(wrapped(err))).toBe(true);+ });+ });++ describe("other β transient / infra / unknown (bias toward investigation)", () => {+ it.each(["ETIMEDOUT", "ECONNRESET", "EAI_AGAIN", "ECONNREFUSED", "EPIPE"])(+ "classifies network error %s as other",+ (code) => {+ const err = new Error(`network ${code}`);+ Object.assign(err, { code });+ expect(isCustomerFaultError(wrapped(err))).toBe(false);+ },+ );++ it("classifies a 503 ServiceUnavailable as other", () => {+ expect(+ isCustomerFaultError(wrapped(s3Error("ServiceUnavailable", 503))),+ ).toBe(false);+ });++ it("classifies a 500 InternalError as other", () => {+ expect(isCustomerFaultError(wrapped(s3Error("InternalError", 500)))).toBe(+ false,+ );+ });++ it("classifies S3 SlowDown / throttling as other", () => {+ expect(isCustomerFaultError(wrapped(s3Error("SlowDown", 503)))).toBe(+ false,+ );+ });++ it("classifies a bare 403 with no recognized code as other (e.g. clock skew)", () => {+ expect(+ isCustomerFaultError(wrapped(s3Error("RequestTimeTooSkewed", 403))),+ ).toBe(false);+ });++ it("classifies a RequestTimeout as other", () => {+ expect(+ isCustomerFaultError(wrapped(s3Error("RequestTimeout", 400))),+ ).toBe(false);+ });++ it("classifies a plain application error as other", () => {+ expect(+ isCustomerFaultError(+ new Error("The configured export source includes enriched ..."),+ ),+ ).toBe(false);+ });++ it("classifies a GCS object-level notFound as other", () => {+ expect(isCustomerFaultError(wrapped(gcsError(404, "notFound")))).toBe(+ false,+ );+ });++ it("classifies null/undefined/non-error inputs as other", () => {+ expect(isCustomerFaultError(null)).toBe(false);+ expect(isCustomerFaultError(undefined)).toBe(false);+ expect(isCustomerFaultError("AccessDenied")).toBe(false);+ expect(isCustomerFaultError(403)).toBe(false);+ });+ });++ describe("customer_fault β outbound-URL / SSRF validation rejection", () => {+ it.each(OUTBOUND_URL_FAULTS.map(([code]) => code))(+ "classifies a %s validation rejection as customer_fault",+ (code) => {+ const err = new OutboundUrlValidationError(code, "blocked");+ expect(isCustomerFaultError(err)).toBe(true);+ expect(isCustomerFaultError(wrapped(err))).toBe(true);+ },+ );++ it("classifies dns-lookup-failed as other, not customer_fault", () => {+ // Resolvability depends on runtime resolver state, not the endpoint+ // config: a transient DNS outage across the retry window must not+ // permanently disable a working integration. Keep it investigable,+ // consistent with the network EAI_AGAIN case above.+ const err = new OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed for host.example",+ );+ expect(isCustomerFaultError(err)).toBe(false);+ expect(isCustomerFaultError(wrapped(err))).toBe(false);+ });++ it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {+ // Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying+ // the same deterministic code, with no `cause` chained.+ const rewrapped = new OutboundUrlValidationError(+ "blocked-ip",+ "Blocked IP address detected For self-hosted deployments ...",+ );+ expect(isCustomerFaultError(wrapped(rewrapped))).toBe(true);+ });++ it("does NOT classify a code-less error that merely shares the message as customer_fault", () => {+ // Classification is code-based, not message-based: a bare 'DNS lookup+ // failed' Error (e.g. some future unrelated throw site) stays investigable.+ const err = new Error("DNS lookup failed for host.example");+ expect(isCustomerFaultError(wrapped(err))).toBe(false);+ });+ });++ describe("cause-chain traversal", () => {+ it("finds a customer fault nested several causes deep", () => {+ const root = s3Error("InvalidAccessKeyId", 403);+ const mid = new Error("mid", { cause: root });+ const top = new Error("top", { cause: mid });+ expect(isCustomerFaultError(top)).toBe(true);+ });++ it("does not infinite-loop on a cyclic cause chain", () => {+ const a = new Error("a");+ const b = new Error("b", { cause: a });+ Object.assign(a, { cause: b });+ expect(isCustomerFaultError(a)).toBe(false);+ });+ });+});++describe("classifyCustomerFault β disable reason buckets", () => {+ it.each(OUTBOUND_URL_FAULTS)("maps outbound-url %s to %s", (code, reason) => {+ const err = new OutboundUrlValidationError(code, "blocked");+ expect(classifyCustomerFault(err)).toBe(reason);+ expect(classifyCustomerFault(wrapped(err))).toBe(reason);+ });++ it.each(["InvalidAccessKeyId", "AccessDenied", "SignatureDoesNotMatch"])(+ "maps S3 auth code %s to credentials",+ (code) => {+ expect(classifyCustomerFault(wrapped(s3Error(code, 403)))).toBe(+ "credentials",+ );+ },+ );++ it.each(["AuthenticationFailed", "InsufficientAccountPermissions"])(+ "maps Azure auth code %s to credentials",+ (code) => {+ expect(classifyCustomerFault(wrapped(azureError(code, 403)))).toBe(+ "credentials",+ );+ },+ );++ it("maps a bare 401 and GCS forbidden to credentials", () => {+ const unauthorized = new Error("Unauthorized");+ Object.assign(unauthorized, { $metadata: { httpStatusCode: 401 } });+ expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");+ expect(classifyCustomerFault(wrapped(gcsError(403, "forbidden")))).toBe(+ "credentials",+ );+ });++ it.each([+ ["NoSuchBucket", 404],+ ["InvalidBucketName", 400],+ ["ContainerNotFound", 404],+ ["InvalidResourceName", 400],+ ] as const)("maps %s to bucket_or_container", (code, status) => {+ // extractErrorCodes reads .name/.Code/.code, so the code bucket is+ // provider-agnostic here β the s3Error shape is enough to exercise it.+ expect(classifyCustomerFault(wrapped(s3Error(code, status)))).toBe(+ "bucket_or_container",+ );+ });++ it("returns undefined for non-disabling errors (dns-lookup-failed, transient)", () => {+ expect(+ classifyCustomerFault(+ new OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed",+ ),+ ),+ ).toBeUndefined();+ const transient = new Error("network EAI_AGAIN");+ Object.assign(transient, { code: "EAI_AGAIN" });+ expect(classifyCustomerFault(wrapped(transient))).toBeUndefined();+ });++ // The reason bucket doubles as the disable decision (defined => disable), so+ // every non-disabling input must yield undefined and not merely a falsy+ // `isCustomerFaultError`. Covers the wrapped and bare-input paths that the+ // boolean suite above asserts only through isCustomerFaultError.+ describe("undefined for every non-disabling input", () => {+ it("returns undefined for a wrapped dns-lookup-failed", () => {+ const err = new OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed for host.example",+ );+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();+ });++ it("returns undefined for a generic infra error, raw and wrapped", () => {+ const err = new Error("ClickHouse read failed");+ expect(classifyCustomerFault(err)).toBeUndefined();+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();+ });++ it("returns undefined for a code-less error sharing a fault message", () => {+ const err = new Error("Blocked hostname detected");+ expect(classifyCustomerFault(err)).toBeUndefined();+ expect(classifyCustomerFault(wrapped(err))).toBeUndefined();+ });++ it("returns undefined for null, undefined, and non-error inputs", () => {+ expect(classifyCustomerFault(null)).toBeUndefined();+ expect(classifyCustomerFault(undefined)).toBeUndefined();+ expect(classifyCustomerFault("blocked-hostname")).toBeUndefined();+ expect(classifyCustomerFault(403)).toBeUndefined();+ });+ });+});diff --git a/worker/src/features/integrations/customerFaultClassification.ts b/worker/src/features/integrations/customerFaultClassification.tsnew file mode 100644index 0000000000..e24f79c85b--- /dev/null+++ b/worker/src/features/integrations/customerFaultClassification.ts@@ -0,0 +1,177 @@+// Classifies a caught integration-export error as a deterministic+// customer-config / credential fault. Integration handlers use this to decide+// whether to auto-disable an integration instead of retrying/alerting forever.+// Each feature owns its own disable metric; only the classification is shared+// here.+//+// Conservative allowlist of stable provider error codes, biased toward false:+// a false positive disables a working integration, a false negative only keeps+// retrying. Errors arrive wrapped via `new Error(..., { cause })` (e.g.+// StorageService.handleStorageError), so we walk the cause chain.++// Coarse cause bucket for a disable, for logs/metrics. `ssrf_blocked_endpoint`+// is the SSRF-guard rejection (endpoint resolves to a blocked host/IP) β the+// "likely abuse" bucket to watch β while the rest are customer misconfiguration.+export type CustomerFaultReason =+ | "ssrf_blocked_endpoint"+ | "invalid_endpoint_url"+ | "credentials"+ | "bucket_or_container";++// AWS SDK v3 sets `.name`/`.Code`; Azure RestError sets `.code`. S3-compatible+// providers (incl. GCS S3-interop) surface the S3 codes.+const CREDENTIAL_FAULT_CODES = new Set<string>([+ // S3 β auth & credentials+ "InvalidAccessKeyId",+ "SignatureDoesNotMatch",+ "AccessDenied",+ "AllAccessDisabled",+ "AccountProblem",+ "AuthorizationHeaderMalformed",+ // Azure β auth & credentials+ "AuthenticationFailed",+ "AuthorizationFailure",+ "AuthorizationPermissionMismatch",+ "InvalidAuthenticationInfo",+ "AccountIsDisabled",+ "InsufficientAccountPermissions",+]);++const BUCKET_FAULT_CODES = new Set<string>([+ // S3 β bucket & path+ "NoSuchBucket",+ "InvalidBucketName",+ // Azure β container & path+ "ContainerNotFound",+ "InvalidResourceName",+]);++// Langfuse outbound-URL / SSRF validation rejections (OutboundUrlValidationError+// from @langfuse/shared/.../outbound-url), split by cause. Every code here is a+// deterministic property of the endpoint config, so it is safe to auto-disable+// on. We deliberately omit `dns-lookup-failed`: resolvability depends on runtime+// resolver state, not the config, so a transient DNS outage across the retry+// window must not permanently disable a working integration. secureLlmFetch+// makes the same distinction (dns-lookup-failed -> "endpoint-unreachable").+const SSRF_BLOCKED_OUTBOUND_URL_CODES = new Set<string>([+ "blocked-hostname",+ "blocked-ip",+]);++const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>([+ "invalid-syntax",+ "invalid-encoding",+ "https-required",+ "protocol-not-allowed",+ "url-credentials-not-allowed",+]);++// GCS JSON-API reasons (only reachable via a GCS-native client; S3-interop uses+// the S3 codes above). Narrow on purpose: object-level notFound stays `other`.+const CUSTOMER_FAULT_GCS_REASONS = new Set<string>(["forbidden"]);++const MAX_CAUSE_DEPTH = 10; // guard against cyclic `cause`++function* errorCauseChain(error: unknown): Generator<object> {+ let current: unknown = error;+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {+ if (!current || typeof current !== "object") return;+ yield current;+ current = (current as { cause?: unknown }).cause;+ }+}++// `.code` is overloaded by type: a number is a GCS HTTP status (here), a string+// is an Azure error code (extractErrorCodes). Split the reads if a future+// provider ever reports a numeric `code: 401` that isn't a credential failure.+function extractHttpStatus(err: object): number | undefined {+ const metaStatus = (err as { $metadata?: { httpStatusCode?: unknown } })+ .$metadata?.httpStatusCode;+ if (typeof metaStatus === "number") return metaStatus;++ const statusCode = (err as { statusCode?: unknown }).statusCode;+ if (typeof statusCode === "number") return statusCode;++ const numericCode = (err as { code?: unknown }).code;+ if (typeof numericCode === "number") return numericCode;++ return undefined;+}++function extractErrorCodes(err: object): string[] {+ const candidates = [+ (err as { name?: unknown }).name,+ (err as { Code?: unknown }).Code,+ (err as { code?: unknown }).code,+ ];+ return candidates.filter((c): c is string => typeof c === "string");+}++function extractGcsReasons(err: object): string[] {+ const errors = (err as { errors?: unknown }).errors;+ if (!Array.isArray(errors)) return [];+ return errors+ .map((e) =>+ e &&+ typeof e === "object" &&+ typeof (e as { reason?: unknown }).reason === "string"+ ? (e as { reason: string }).reason+ : undefined,+ )+ .filter((r): r is string => typeof r === "string");+}++// Duck-type on `.name` (survives cross-package module duplication where+// `instanceof` is unreliable) rather than importing the class, then map the+// deterministic code to a reason; transient reasons like `dns-lookup-failed`+// return undefined so they stay non-disabling.+function classifyOutboundUrlFault(+ err: object,+): CustomerFaultReason | undefined {+ if ((err as { name?: unknown }).name !== "OutboundUrlValidationError") {+ return undefined;+ }+ const code = (err as { code?: unknown }).code;+ if (typeof code !== "string") return undefined;+ if (SSRF_BLOCKED_OUTBOUND_URL_CODES.has(code)) return "ssrf_blocked_endpoint";+ if (INVALID_URL_OUTBOUND_URL_CODES.has(code)) return "invalid_endpoint_url";+ return undefined;+}++function classifyCustomerFaultLink(+ err: object,+): CustomerFaultReason | undefined {+ const outbound = classifyOutboundUrlFault(err);+ if (outbound) return outbound;++ const codes = extractErrorCodes(err);+ if (codes.some((c) => CREDENTIAL_FAULT_CODES.has(c))) return "credentials";+ if (codes.some((c) => BUCKET_FAULT_CODES.has(c)))+ return "bucket_or_container";++ // 401 = credentials rejected: unambiguous, so it trips without a code. A bare+ // 403/404 does not (e.g. clock-skew RequestTimeTooSkewed is a 403).+ if (extractHttpStatus(err) === 401) return "credentials";+ if (extractGcsReasons(err).some((r) => CUSTOMER_FAULT_GCS_REASONS.has(r))) {+ return "credentials";+ }+ return undefined;+}++// Returns the deterministic customer-fault reason for the first matching link in+// the cause chain, or undefined when nothing qualifies (retry-worthy / infra /+// transient). The handler disables on any defined reason and tags the disable+// log + metric with it.+export function classifyCustomerFault(+ error: unknown,+): CustomerFaultReason | undefined {+ for (const link of errorCauseChain(error)) {+ const reason = classifyCustomerFaultLink(link);+ if (reason) return reason;+ }+ return undefined;+}++export function isCustomerFaultError(error: unknown): boolean {+ return classifyCustomerFault(error) !== undefined;+}diff --git a/worker/src/features/integrations/prismaErrors.ts b/worker/src/features/integrations/prismaErrors.tsnew file mode 100644index 0000000000..032c35b2dc--- /dev/null+++ b/worker/src/features/integrations/prismaErrors.ts@@ -0,0 +1,8 @@+import { Prisma } from "@langfuse/shared/src/db";++// An integration can be deleted while one of its runs is still in flight; a row+// write racing that delete throws P2025. Integration handlers treat that as+// "job is obsolete, complete it quietly" rather than as a failure to retry.+export const isRecordNotFoundError = (error: unknown): boolean =>+ error instanceof Prisma.PrismaClientKnownRequestError &&+ error.code === "P2025";