🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): fail fast and auto-disable the PostHog export on config faults (#16095)
Niklas Semmler6387ef983325
diff --git a/packages/shared/src/server/outbound-url/validation.ts b/packages/shared/src/server/outbound-url/validation.tsindex 8ab6bef33a..f0fc6535f1 100644--- a/packages/shared/src/server/outbound-url/validation.ts+++ b/packages/shared/src/server/outbound-url/validation.ts@@ -14,6 +14,7 @@ export type OutboundUrlValidationErrorCode =| "https-required"| "invalid-encoding"| "invalid-syntax"+ | "port-not-allowed"| "protocol-not-allowed"| "url-credentials-not-allowed";diff --git a/packages/shared/src/server/webhooks/validation.ts b/packages/shared/src/server/webhooks/validation.tsindex 1aa0daa5ae..bc3e1c564b 100644--- a/packages/shared/src/server/webhooks/validation.ts+++ b/packages/shared/src/server/webhooks/validation.ts@@ -1,5 +1,6 @@import { env } from "../../env";import {+ OutboundUrlValidationError,type OutboundUrlValidationWhitelist,parseOutboundUrl,resolveHost,@@ -39,13 +40,24 @@ export async function validateWebhookURL(): Promise<void> {const url = parseOutboundUrl(urlString);+ // Coded, not bare Error: callers classify rejections off the `code` rather+ // than the message. validateBlobStorageEndpoint already does this for the+ // protocol check; these two were the only outbound-URL rejections in the repo+ // that reached callers uncoded, which silently excluded them from+ // customer-fault classification. Messages are unchanged.if (!["https:", "http:"].includes(url.protocol)) {- throw new Error("Only HTTP and HTTPS protocols are allowed");+ throw new OutboundUrlValidationError(+ "protocol-not-allowed",+ "Only HTTP and HTTPS protocols are allowed",+ );}const allowedPorts = options.allowedPorts ?? DEFAULT_WEBHOOK_ALLOWED_PORTS;if (url.port && allowedPorts !== "any" && !allowedPorts.includes(url.port)) {- throw new Error("Only ports 80 and 443 are allowed");+ throw new OutboundUrlValidationError(+ "port-not-allowed",+ "Only ports 80 and 443 are allowed",+ );}await validateOutboundUrlHost({diff --git a/web/src/features/notifications/components/ProjectNotificationChannels.tsx b/web/src/features/notifications/components/ProjectNotificationChannels.tsxindex 76d022ae56..ae40f282a2 100644--- a/web/src/features/notifications/components/ProjectNotificationChannels.tsx+++ b/web/src/features/notifications/components/ProjectNotificationChannels.tsx@@ -35,6 +35,12 @@ const NOTIFIED_EVENTS: {title: "Blob storage export failed",description: "Sent when a scheduled blob storage export fails.",},+ {+ value: "posthog-export-failed",+ title: "PostHog export failed",+ description:+ "Sent when a PostHog export is disabled after a configuration error, such as an unreachable host.",+ },{value: "evaluator-blocked",title: "Evaluator deactivated",diff --git a/worker/src/__tests__/posthogIntegrationProjectJob.test.ts b/worker/src/__tests__/posthogIntegrationProjectJob.test.tsindex b557dafb7c..e325984abb 100644--- a/worker/src/__tests__/posthogIntegrationProjectJob.test.ts+++ b/worker/src/__tests__/posthogIntegrationProjectJob.test.ts@@ -51,12 +51,72 @@ const h = vi.hoisted(() => {})();}- const posthogIntegrationUpdate = vi.fn();+ // How many rows the atomic disable claim matches. 0 simulates a lost claim:+ // either a concurrent run already flipped enabled, or the customer corrected+ // the hostname mid-run so the run's `where` predicate no longer matches.+ const disableClaim = { matchedRows: 1 };++ // Observable completion of the terminal notification. `settled` flips only+ // after a macrotask, so a caller that fires the dispatch without awaiting it+ // returns while `settled` is still false — that gap is what distinguishes an+ // awaited dispatch from a fire-and-forget one.+ const notification = { settled: false, rejects: false };++ // Stand-in for the Prisma namespace class. The record-not-found predicate is+ // instanceof-based, so a duck-typed `{ code: "P2025" }` would not satisfy it+ // — tests must throw a real instance of this class.+ class PrismaClientKnownRequestError extends Error {+ code: string;+ constructor(message: string, { code }: { code: string }) {+ super(message);+ this.name = "PrismaClientKnownRequestError";+ this.code = code;+ }+ }++ // Lets a test fail the lastError/lastErrorAt persist write without disturbing+ // the tests that need it to resolve.+ const persist = { failWith: undefined as unknown };++ const posthogIntegrationUpdate = vi.fn(async () => {+ if (persist.failWith !== undefined) throw persist.failWith;+ });+ // The atomic disable (enabled true->false) uses updateMany. Only the disable+ // write is claim-controlled; any other updateMany keeps matching.+ const posthogIntegrationUpdateMany = vi.fn(+ async (args?: { data?: Record<string, unknown> }) => ({+ count: args?.data?.enabled === false ? disableClaim.matchedRows : 1,+ }),+ );+ // Hostname preflight (throws OutboundUrlValidationError on a bad+ // hostname) and the customer notification, both controllable per test.+ const validateWebhookURL = vi.fn(async () => {});+ const recordIncrement = vi.fn();+ const dispatchProjectNotification = vi.fn(async () => {+ await new Promise((resolve) => setTimeout(resolve, 0));+ try {+ if (notification.rejects) throw new Error("notification dispatch failed");+ } finally {+ notification.settled = true;+ }+ });const getTraces = vi.fn(() => fakeStream("traces"));const getGenerations = vi.fn(() => fakeStream("generations"));const getScores = vi.fn(() => fakeStream("scores"));const getEvents = vi.fn(() => fakeStream("events"));+ // Minimal stand-in for the shared class. The classifier duck-types on+ // `.name === "OutboundUrlValidationError"` + `.code` (it does not use+ // instanceof), so this shape is sufficient for the mocked server barrel.+ class OutboundUrlValidationError extends Error {+ code: string;+ constructor(code: string, message: string) {+ super(message);+ this.name = "OutboundUrlValidationError";+ this.code = code;+ }+ }+const defaultIntegration = () => ({projectId: "project-1",enabled: true,@@ -76,6 +136,15 @@ const h = vi.hoisted(() => {constructed,state,posthogIntegrationUpdate,+ posthogIntegrationUpdateMany,+ PrismaClientKnownRequestError,+ persist,+ disableClaim,+ notification,+ validateWebhookURL,+ recordIncrement,+ dispatchProjectNotification,+ OutboundUrlValidationError,getTraces,getGenerations,getScores,@@ -90,16 +159,25 @@ vi.mock("@langfuse/shared/src/db", () => ({posthogIntegration: {findFirst: vi.fn(async () => h.db.integration),update: h.posthogIntegrationUpdate,+ updateMany: h.posthogIntegrationUpdateMany,},},+ // The record-not-found predicate lives in its own module but imports Prisma+ // from this specifier, so this factory intercepts it too. Without this+ // export the predicate throws a TypeError instead of returning a boolean.+ Prisma: {+ PrismaClientKnownRequestError: h.PrismaClientKnownRequestError,+ },}));vi.mock("@langfuse/shared/src/server", () => ({QueueName: { PostHogIntegrationProcessingQueue: "posthog" },logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },- recordIncrement: vi.fn(),+ recordIncrement: h.recordIncrement,getCurrentSpan: vi.fn(() => undefined),- validateWebhookURL: vi.fn(async () => {}),+ validateWebhookURL: h.validateWebhookURL,+ dispatchProjectNotification: h.dispatchProjectNotification,+ OutboundUrlValidationError: h.OutboundUrlValidationError,getTracesForAnalyticsIntegrations: h.getTraces,getGenerationsForAnalyticsIntegrations: h.getGenerations,getScoresForAnalyticsIntegrations: h.getScores,@@ -191,6 +269,17 @@ function resetSharedState() {h.state.rowCounts.generations = 2;h.state.rowCounts.events = 2;h.posthogIntegrationUpdate.mockClear();+ h.persist.failWith = undefined;+ // mockClear, not mockReset: the implementations above are claim/notification+ // aware and must survive between tests; only the recorded calls reset.+ h.posthogIntegrationUpdateMany.mockClear();+ h.disableClaim.matchedRows = 1;+ h.notification.settled = false;+ h.notification.rejects = false;+ h.recordIncrement.mockClear();+ h.validateWebhookURL.mockReset();+ h.validateWebhookURL.mockImplementation(async () => {});+ h.dispatchProjectNotification.mockClear();h.getTraces.mockClear();h.getGenerations.mockClear();h.getScores.mockClear();@@ -425,3 +514,282 @@ describe("handlePostHogIntegrationProjectJob delivery controls (issue #12786)",expect(h.posthogIntegrationUpdate).not.toHaveBeenCalled();});});++// A deterministic customer-config fault (bad/malicious hostname+// rejected by validateWebhookURL) must be classified as the customer's problem+// — the job RESOLVES (not thrown, so the type:failed counter never increments+// and the Failures monitor stays quiet), the integration is auto-disabled,+// lastError/lastErrorAt are persisted, and the customer is notified once with+// disabled=true. Transient/infra faults still throw, retry, and trip the+// monitor unchanged.+describe("handlePostHogIntegrationProjectJob customer-fault auto-disable", () => {+ beforeEach(() => {+ resetSharedState();+ // A known-good source + write-mode combo so the pre-export write-mode+ // guards pass and the handler reaches the hostname preflight.+ h.db.integration = {+ ...h.defaultIntegration(),+ exportSource: "TRACES_OBSERVATIONS",+ };+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";+ });++ // Every `data` payload written across update + updateMany, so assertions+ // don't over-constrain which Prisma method carries which column.+ const writeData = (): Array<Record<string, unknown>> =>+ [+ ...h.posthogIntegrationUpdate.mock.calls,+ ...h.posthogIntegrationUpdateMany.mock.calls,+ ].map(+ (call) =>+ (call[0] as { data?: Record<string, unknown> } | undefined)?.data ?? {},+ );++ const disableNotifications = (): number =>+ h.dispatchProjectNotification.mock.calls.filter((call) => {+ const arg = call[0] as { event?: { disabled?: unknown } } | undefined;+ return arg?.event?.disabled === true;+ }).length;++ // The atomic disable writes: updateMany calls carrying `enabled: false`.+ const disableWrites = (): Array<{+ where?: Record<string, unknown>;+ data?: Record<string, unknown>;+ }> =>+ h.posthogIntegrationUpdateMany.mock.calls+ .map((call) => call[0] as { where?: any; data?: any } | undefined)+ .filter((arg): arg is { where?: any; data?: any } => Boolean(arg))+ .filter((arg) => arg.data?.enabled === false);++ // Counter emitted when an integration is auto-disabled. Matched on the+ // metric-name fragment rather than the full string so the assertion tracks+ // the behavior, not the namespace. The winning-claim test below asserts this+ // filter yields exactly one call, which keeps the "not recorded" assertions+ // from passing vacuously if the metric were ever renamed away.+ const disableMetrics = (): unknown[][] =>+ h.recordIncrement.mock.calls.filter(+ ([name]) =>+ typeof name === "string" && name.includes("integration_disabled"),+ );++ const badHostnameFault = () =>+ new h.OutboundUrlValidationError(+ "blocked-hostname",+ "Blocked hostname detected",+ );++ // Load-bearing: the core journey. A blocked-hostname fault must resolve (no+ // throw) AND flip enabled to false AND persist the error AND notify once.+ it("resolves and auto-disables on a blocked-hostname validation fault", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(+ new h.OutboundUrlValidationError(+ "blocked-hostname",+ "Blocked hostname detected",+ ),+ );++ // Must not throw — a throw would increment type:failed and fire the monitor.+ await handlePostHogIntegrationProjectJob(makeJob());++ expect(writeData().some((data) => data.enabled === false)).toBe(true);+ expect(+ writeData().some(+ (data) =>+ typeof data.lastError === "string" &&+ (data.lastError as string).length > 0,+ ),+ ).toBe(true);+ expect(writeData().some((data) => data.lastErrorAt instanceof Date)).toBe(+ true,+ );+ expect(disableNotifications()).toBe(1);+ });++ // Load-bearing: the throw site rewraps into `new Error(msg, { cause })`, so+ // the handler must classify off the cause chain — a wrapped validation error+ // still disables.+ it("classifies through a wrapped cause chain and still disables", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(+ new Error("posthog export failed", {+ cause: new h.OutboundUrlValidationError(+ "blocked-hostname",+ "Blocked hostname detected",+ ),+ }),+ );++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(writeData().some((data) => data.enabled === false)).toBe(true);+ expect(disableNotifications()).toBe(1);+ });++ // A disallowed protocol or port is as deterministic as a blocked hostname —+ // no retry can fix `ftp://` or `:8080`. These reach the handler as coded+ // errors only because validateWebhookURL raises OutboundUrlValidationError+ // for them; while it threw a bare Error, they were unclassifiable and+ // retried forever. Real coded-vs-uncoded coverage lives in+ // webhook-validation.test.ts, since this suite mocks the validator.+ it.each([+ ["protocol-not-allowed", "Only HTTP and HTTPS protocols are allowed"],+ ["port-not-allowed", "Only ports 80 and 443 are allowed"],+ ] as const)("resolves and auto-disables on %s", async (code, message) => {+ h.validateWebhookURL.mockRejectedValueOnce(+ new h.OutboundUrlValidationError(code, message),+ );++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(writeData().some((data) => data.enabled === false)).toBe(true);+ expect(disableNotifications()).toBe(1);+ });++ // Load-bearing negative controls: transient/infra faults are NOT customer+ // faults — the handler must rethrow (so BullMQ retries and the monitor+ // fires), leave enabled unchanged, and never notify.+ it.each([+ [+ "a dns-lookup-failed validation fault",+ () =>+ new h.OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed for host.example",+ ),+ ],+ [+ "a generic ClickHouse-style error",+ () => new Error("ClickHouse read failed"),+ ],+ ] as const)(+ "rethrows on %s, leaves enabled unchanged, and does not notify",+ async (_label, makeError) => {+ h.validateWebhookURL.mockRejectedValueOnce(makeError());++ await expect(+ handlePostHogIntegrationProjectJob(makeJob()),+ ).rejects.toThrow();++ expect(writeData().some((data) => data.enabled === false)).toBe(false);+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();+ },+ );++ // Cheap addition (Semantics table success row): a healthy run clears any+ // prior error state so a fixed + re-enabled integration reads as healthy.+ it("clears lastError/lastErrorAt on a successful run", async () => {+ await handlePostHogIntegrationProjectJob(makeJob());++ expect(h.posthogIntegrationUpdate).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({ lastError: null, lastErrorAt: null }),+ }),+ );+ });++ // Load-bearing: the disable predicate must pin the hostname this run actually+ // validated. Without it, a run carrying stale config would disable a hostname+ // the customer already corrected mid-run. Asserted on the real argument, so a+ // predicate that silently drops the host guard goes red.+ it("scopes the atomic disable to the hostname this run validated", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(disableWrites()).toHaveLength(1);+ expect(disableWrites()[0].where).toEqual({+ projectId: "project-1",+ enabled: true,+ posthogHostName: "https://us.posthog.com",+ });+ });++ // Positive control for the two "not recorded / not notified" assertions+ // below: proves the disable-metric filter and the notification counter both+ // observe something real on the winning path.+ it("records the disable metric and notifies once when the claim is won", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(disableMetrics()).toHaveLength(1);+ expect(disableNotifications()).toBe(1);+ });++ // Load-bearing: the notify-once guarantee under a lost claim. count === 0+ // means either a concurrent run already flipped enabled or the customer+ // corrected the hostname mid-run; in both cases this run must stay silent+ // rather than claim a state it did not write.+ it("stays silent when the disable claim matches no row", async () => {+ h.disableClaim.matchedRows = 0;+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());++ // Still a deliberate resolve: a lost claim is not a job failure.+ await handlePostHogIntegrationProjectJob(makeJob());++ expect(disableWrites()).toHaveLength(1);+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();+ expect(disableMetrics()).toHaveLength(0);+ });++ // Load-bearing: fail-fast resolves the job and the scheduler only enqueues+ // enabled integrations, so a dispatch left in flight has no retry path and+ // would leave the customer silently disabled. The notification mock settles+ // one macrotask late, so this flag is still false if the dispatch is fired+ // without being awaited.+ it("completes the disable notification before the handler resolves", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(h.notification.settled).toBe(true);+ });++ // Load-bearing: awaiting the dispatch must not turn the deliberate resolve+ // into a job failure — the notify helper swallows its own errors, and the+ // disable stands regardless of whether the customer could be reached.+ it("resolves and keeps the integration disabled when the notification rejects", async () => {+ h.notification.rejects = true;+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(disableWrites()).toHaveLength(1);+ expect(h.notification.settled).toBe(true);+ });++ // Load-bearing: the integration can be deleted while the run is in flight, so+ // the persist races the delete and throws P2025. There is nothing to persist+ // and nobody to notify — the job is obsolete, so it must resolve quietly+ // rather than disable a row that no longer exists or fail the job.+ it("drops the obsolete job when the integration is deleted mid-run", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());+ h.persist.failWith = new h.PrismaClientKnownRequestError(+ "An operation failed because it depends on one or more records that were required but not found.",+ { code: "P2025" },+ );++ await handlePostHogIntegrationProjectJob(makeJob());++ expect(h.posthogIntegrationUpdate).toHaveBeenCalledTimes(1);+ expect(disableWrites()).toHaveLength(0);+ expect(disableMetrics()).toHaveLength(0);+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();+ });++ // Load-bearing contrast to the case above: any other persist failure means we+ // do NOT know the fault was recorded, so the run must not claim it as handled+ // — rethrow and let BullMQ retry and reclassify. Without this, a swallowed DB+ // outage would silently drop the customer fault.+ it("rethrows when persisting the error fails for any other reason", async () => {+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());+ h.persist.failWith = new Error("database unavailable");++ await expect(+ handlePostHogIntegrationProjectJob(makeJob()),+ ).rejects.toThrow();++ expect(disableWrites()).toHaveLength(0);+ expect(disableMetrics()).toHaveLength(0);+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/__tests__/webhook-validation.test.ts b/worker/src/__tests__/webhook-validation.test.tsindex e3b14ab501..0d99a20f53 100644--- a/worker/src/__tests__/webhook-validation.test.ts+++ b/worker/src/__tests__/webhook-validation.test.ts@@ -88,6 +88,22 @@ describe("Webhook URL Validation", () => {).rejects.toThrow("Only ports 80 and 443 are allowed");});+ // Protocol and port rejections must carry an OutboundUrlValidationError+ // code, not just a message: callers classify deterministic config faults+ // off the code, so an uncoded rejection is silently unclassifiable and+ // retries forever instead of disabling the integration.+ it.each([+ ["ftp://example.com", "protocol-not-allowed"],+ ["file:///etc/passwd", "protocol-not-allowed"],+ ["https://example.com:8080/hook", "port-not-allowed"],+ ["http://example.com:3000/hook", "port-not-allowed"],+ ])("should reject %s with a coded error", async (url, code) => {+ await expect(validateWebhookURL(url)).rejects.toMatchObject({+ name: "OutboundUrlValidationError",+ code,+ });+ });+it("should allow standard ports", async () => {await expect(validateWebhookURL("https://httpbin.org:443/post"),diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.tsindex 6de62a4ec1..9b09a2aead 100644--- a/worker/src/features/integrations/customerFaultClassification.test.ts+++ b/worker/src/features/integrations/customerFaultClassification.test.ts@@ -56,6 +56,7 @@ const OUTBOUND_URL_FAULTS = [["invalid-encoding", "invalid_endpoint_url"],["https-required", "invalid_endpoint_url"],["protocol-not-allowed", "invalid_endpoint_url"],+ ["port-not-allowed", "invalid_endpoint_url"],["url-credentials-not-allowed", "invalid_endpoint_url"],] as const;diff --git a/worker/src/features/integrations/customerFaultClassification.ts b/worker/src/features/integrations/customerFaultClassification.tsindex e24f79c85b..216752187b 100644--- a/worker/src/features/integrations/customerFaultClassification.ts+++ b/worker/src/features/integrations/customerFaultClassification.ts@@ -63,6 +63,7 @@ const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>(["invalid-encoding","https-required","protocol-not-allowed",+ "port-not-allowed","url-credentials-not-allowed",]);diff --git a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.tsindex 9ef9f637b5..d6777da978 100644--- a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts+++ b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts@@ -10,6 +10,8 @@ import {getEventsForAnalyticsIntegrations,getCurrentSpan,validateWebhookURL,+ recordIncrement,+ dispatchProjectNotification,} from "@langfuse/shared/src/server";import {transformTraceForPostHog,@@ -21,8 +23,17 @@ import { decrypt } from "@langfuse/shared/encryption";import { PostHog } from "posthog-node";import { recordExportVolume } from "../../services/exportVolumeMetric";import { assertExportSourceWritable } from "../exportWriteModeGuard";+import { classifyCustomerFault } from "../integrations/customerFaultClassification";+import { isRecordNotFoundError } from "../integrations/prismaErrors";import { env, v4WritesToLegacyTables } from "../../env";+// Counter for PostHog integrations auto-disabled after a deterministic+// customer-config fault, tagged by `reason`. A classifier-regression+// mass-disable shows up as a spike in the non-SSRF buckets vs a near-zero+// baseline, separate from expected SSRF/abuse disables (mirrors blob's metric).+export const POSTHOG_INTEGRATION_DISABLED_METRIC =+ "langfuse.posthog.integration_disabled.count";+type PostHogExecutionConfig = {projectId: string;projectName: string;@@ -258,65 +269,70 @@ export const handlePostHogIntegrationProjectJob = async (return;}- // Validate PostHog hostname to prevent SSRF attacks before sending datatry {- await validateWebhookURL(postHogIntegration.posthogHostName);- } catch (error) {- logger.error(- `[POSTHOG] PostHog integration for project ${projectId} has invalid hostname: ${postHogIntegration.posthogHostName}. Error: ${error instanceof Error ? error.message : String(error)}`,+ // Validate PostHog hostname to prevent SSRF attacks before sending data.+ // Rewrap preserving { cause } so the single catch below can classify the+ // OutboundUrlValidationError off the chain.+ try {+ await validateWebhookURL(postHogIntegration.posthogHostName);+ } catch (error) {+ logger.error(+ `[POSTHOG] PostHog integration for project ${projectId} has invalid hostname: ${postHogIntegration.posthogHostName}. Error: ${error instanceof Error ? error.message : String(error)}`,+ );+ throw new Error(+ `Invalid PostHog hostname for project ${projectId}: ${error instanceof Error ? error.message : "Unknown error"}`,+ { cause: error },+ );+ }++ // Resume from lastSyncAt. On first run, fall back to the project's+ // createdAt since no trace data can precede it.+ const minTimestamp =+ postHogIntegration.lastSyncAt || postHogIntegration.project.createdAt;+ const uncappedMaxTimestamp = new Date(Date.now() - 30 * 60 * 1000); // 30 minutes ago++ // Cap maxTimestamp at the next UTC day boundary after minTimestamp. Bounds+ // per-run work so a stuck integration (or a backfill on an older project)+ // does not re-scan an ever-growing window on each hourly retry, and aligns+ // with the toDate(...) ClickHouse partition/ordering keys for better+ // pruning. Healthy integrations are unaffected because uncappedMaxTimestamp+ // wins whenever the sync is within one day of present.+ const nextDayBoundary = new Date(+ Date.UTC(+ minTimestamp.getUTCFullYear(),+ minTimestamp.getUTCMonth(),+ minTimestamp.getUTCDate() + 1,+ ),);- throw new Error(- `Invalid PostHog hostname for project ${projectId}: ${error instanceof Error ? error.message : "Unknown error"}`,+ const maxTimestamp = new Date(+ Math.min(nextDayBoundary.getTime(), uncappedMaxTimestamp.getTime()),);- }- // Resume from lastSyncAt. On first run, fall back to the project's- // createdAt since no trace data can precede it.- const minTimestamp =- postHogIntegration.lastSyncAt || postHogIntegration.project.createdAt;- const uncappedMaxTimestamp = new Date(Date.now() - 30 * 60 * 1000); // 30 minutes ago-- // Cap maxTimestamp at the next UTC day boundary after minTimestamp. Bounds- // per-run work so a stuck integration (or a backfill on an older project)- // does not re-scan an ever-growing window on each hourly retry, and aligns- // with the toDate(...) ClickHouse partition/ordering keys for better- // pruning. Healthy integrations are unaffected because uncappedMaxTimestamp- // wins whenever the sync is within one day of present.- const nextDayBoundary = new Date(- Date.UTC(- minTimestamp.getUTCFullYear(),- minTimestamp.getUTCMonth(),- minTimestamp.getUTCDate() + 1,- ),- );- const maxTimestamp = new Date(- Math.min(nextDayBoundary.getTime(), uncappedMaxTimestamp.getTime()),- );+ if (maxTimestamp <= minTimestamp) {+ logger.info(+ `[POSTHOG] Skipping PostHog integration for project ${projectId}: empty sync window (min: ${minTimestamp.toISOString()}, max: ${maxTimestamp.toISOString()})`,+ );+ return;+ }- if (maxTimestamp <= minTimestamp) {logger.info(- `[POSTHOG] Skipping PostHog integration for project ${projectId}: empty sync window (min: ${minTimestamp.toISOString()}, max: ${maxTimestamp.toISOString()})`,+ `[POSTHOG] Syncing project ${projectId} from ${minTimestamp.toISOString()} to ${maxTimestamp.toISOString()}`,);- return;- }- logger.info(- `[POSTHOG] Syncing project ${projectId} from ${minTimestamp.toISOString()} to ${maxTimestamp.toISOString()}`,- );-- // Fetch relevant data and send it to PostHog- const executionConfig: PostHogExecutionConfig = {- projectId,- projectName: postHogIntegration.project.name,- minTimestamp,- maxTimestamp,- decryptedPostHogApiKey: decrypt(postHogIntegration.encryptedPosthogApiKey),- postHogHost: postHogIntegration.posthogHostName,- useGraceHash: job.attemptsMade > 0,- volume: { bytes: 0 },- };+ // Fetch relevant data and send it to PostHog+ const executionConfig: PostHogExecutionConfig = {+ projectId,+ projectName: postHogIntegration.project.name,+ minTimestamp,+ maxTimestamp,+ decryptedPostHogApiKey: decrypt(+ postHogIntegration.encryptedPosthogApiKey,+ ),+ postHogHost: postHogIntegration.posthogHostName,+ useGraceHash: job.attemptsMade > 0,+ volume: { bytes: 0 },+ };- try {// Fail loudly before exporting empty data and advancing lastSyncAt// (LFE-10148, LFE-11009); the catch below logs and BullMQ retries.assertExportSourceWritable(@@ -376,12 +392,16 @@ export const handlePostHogIntegrationProjectJob = async (throwIfPostHogSendError(executionConfig);// Update the last run information for the postHogIntegration record.+ // Clear any prior error state so a fixed + re-enabled integration reads as+ // healthy.await prisma.posthogIntegration.update({where: {projectId,},data: {lastSyncAt: executionConfig.maxTimestamp,+ lastError: null,+ lastErrorAt: null,},});// Record gzipped on-wire export volume once the run has succeeded.@@ -393,11 +413,128 @@ export const handlePostHogIntegrationProjectJob = async (logger.info(`[POSTHOG] PostHog integration processing complete for project ${projectId}`,);+ } catch (error) {+ // A deterministic customer-config fault (a bad/malicious hostname rejected+ // by validateWebhookURL) can't succeed until the customer fixes it.+ // Classify it, disable the integration, persist the error, notify once,+ // and RESOLVE (return) — a throw would increment the BullMQ `type:failed`+ // metric on every attempt and keep the Failures monitor lit for a fault+ // that will never clear on its own. Transient/infra faults stay on the+ // retry+alert path unchanged.+ const reason = classifyCustomerFault(error);+ // Bounded like blob's extractStorageErrorMessage: today's classified+ // messages are short static strings, but persisted error text should not+ // grow unbounded if a future classified path carries a large SDK body.+ const message = (+ error instanceof Error ? error.message : String(error)+ ).slice(0, 1000);++ if (reason === undefined) {+ logger.error(+ `[POSTHOG] Error processing PostHog integration for project ${projectId}`,+ error,+ );+ throw error;+ }++ try {+ await prisma.posthogIntegration.update({+ where: { projectId },+ data: { lastError: message, lastErrorAt: new Date() },+ });+ } catch (persistError) {+ if (isRecordNotFoundError(persistError)) {+ // Integration deleted mid-run: nothing to persist, nobody to notify.+ // Drop the obsolete job instead of retrying it.+ logger.info(+ `[POSTHOG] PostHog integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${message}`,+ );+ return;+ }+ // Persisting failed for an infra reason (e.g. DB unavailable). Don't claim+ // the customer fault as handled — rethrow so BullMQ retries and reclassifies.+ logger.error(+ `[POSTHOG] Failed to persist PostHog integration error for project ${projectId}`,+ persistError,+ );+ throw persistError;+ }++ // Atomic disable: count === 1 means THIS worker flipped enabled true→false,+ // so the terminal notification fires exactly once even if a concurrent run+ // (distinct jobId, not queue-deduped) races the same fault. The host is+ // part of the predicate so a run carrying stale config cannot disable a+ // hostname the customer corrected mid-run — the fault we caught refers to a+ // config that no longer exists, and the next scheduled run re-evaluates.+ const { count } = await prisma.posthogIntegration.updateMany({+ where: {+ projectId,+ enabled: true,+ posthogHostName: postHogIntegration.posthogHostName,+ },+ data: { enabled: false },+ });+ if (count === 0) {+ // Already disabled by a concurrent run, or the host changed under us.+ logger.info(+ `[POSTHOG] Skipped disabling PostHog integration for project ${projectId}: already disabled or host changed mid-run`,+ );+ return;+ }++ recordIncrement(POSTHOG_INTEGRATION_DISABLED_METRIC, 1, { reason });+ logger.warn(+ `[POSTHOG] Disabled PostHog integration for project ${projectId} after a customer fault (reason=${reason}): ${message}`,+ { posthogDisableReason: reason, projectId },+ );+ // Awaited, not fire-and-forget: the job resolves immediately after this and+ // the integration is now disabled, so the scheduler (which only enqueues+ // enabled integrations) never revisits it. An interrupted dispatch would+ // leave the customer silently disabled with no notification and no retry.+ await notifyPostHogExportFailed(+ projectId,+ postHogIntegration.project.name,+ postHogIntegration.posthogHostName,+ );+ return;+ }+};++// Terminal "export disabled" notification. Called at most once per broken+// integration, guarded by the atomic enabled true→false claim, so it needs no+// cooldown of its own (matching blob's disabled path, which also bypasses it).+// Swallows every failure: notification trouble must not turn the deliberate+// RESOLVE back into a job failure. projectName/host are passed in rather than+// re-read — the caller already holds them.+async function notifyPostHogExportFailed(+ projectId: string,+ projectName: string,+ host: string,+): Promise<void> {+ try {+ const settingsPath = `/project/${projectId}/settings/integrations/posthog`;+ await dispatchProjectNotification({+ projectId,+ event: {+ eventType: "posthog-export-failed",+ severity: "ALERT",+ projectId,+ projectName,+ // The integration is keyed by projectId (1:1); the host is the most+ // useful human label for the failing export destination.+ resourceId: projectId,+ resourceName: host || "PostHog integration",+ message: `PostHog export disabled for project "${projectName}" after a configuration fault.`,+ url: env.NEXTAUTH_URL+ ? `${env.NEXTAUTH_URL}${settingsPath}`+ : undefined,+ disabled: true,+ },+ });} catch (error) {logger.error(- `[POSTHOG] Error processing PostHog integration for project ${projectId}`,+ `[POSTHOG] Failed to send failure notification for project ${projectId}`,error,);- throw error;}-};+}