Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): fail fast and auto-disable the PostHog export on config faults (#16095)

Niklas Semmler6387ef983325
diff --git a/packages/shared/src/server/outbound-url/validation.ts b/packages/shared/src/server/outbound-url/validation.ts
index 8ab6bef33a..f0fc6535f1 100644
--- a/packages/shared/src/server/outbound-url/validation.ts
+++ b/packages/shared/src/server/outbound-url/validation.ts
@@ -14,6 +14,7 @@ export type OutboundUrlValidationErrorCode =
| "https-required"
| "invalid-encoding"
| "invalid-syntax"
+ | "port-not-allowed"
| "protocol-not-allowed"
| "url-credentials-not-allowed";
diff --git a/packages/shared/src/server/webhooks/validation.ts b/packages/shared/src/server/webhooks/validation.ts
index 1aa0daa5ae..bc3e1c564b 100644
--- a/packages/shared/src/server/webhooks/validation.ts
+++ b/packages/shared/src/server/webhooks/validation.ts
@@ -1,5 +1,6 @@
import { env } from "../../env";
import {
+ OutboundUrlValidationError,
type OutboundUrlValidationWhitelist,
parseOutboundUrl,
resolveHost,
@@ -39,13 +40,24 @@ export async function validateWebhookURL(
): Promise<void> {
const url = parseOutboundUrl(urlString);
+ // Coded, not bare Error: callers classify rejections off the `code` rather
+ // than the message. validateBlobStorageEndpoint already does this for the
+ // protocol check; these two were the only outbound-URL rejections in the repo
+ // that reached callers uncoded, which silently excluded them from
+ // customer-fault classification. Messages are unchanged.
if (!["https:", "http:"].includes(url.protocol)) {
- throw new Error("Only HTTP and HTTPS protocols are allowed");
+ throw new OutboundUrlValidationError(
+ "protocol-not-allowed",
+ "Only HTTP and HTTPS protocols are allowed",
+ );
}
const allowedPorts = options.allowedPorts ?? DEFAULT_WEBHOOK_ALLOWED_PORTS;
if (url.port && allowedPorts !== "any" && !allowedPorts.includes(url.port)) {
- throw new Error("Only ports 80 and 443 are allowed");
+ throw new OutboundUrlValidationError(
+ "port-not-allowed",
+ "Only ports 80 and 443 are allowed",
+ );
}
await validateOutboundUrlHost({
diff --git a/web/src/features/notifications/components/ProjectNotificationChannels.tsx b/web/src/features/notifications/components/ProjectNotificationChannels.tsx
index 76d022ae56..ae40f282a2 100644
--- a/web/src/features/notifications/components/ProjectNotificationChannels.tsx
+++ b/web/src/features/notifications/components/ProjectNotificationChannels.tsx
@@ -35,6 +35,12 @@ const NOTIFIED_EVENTS: {
title: "Blob storage export failed",
description: "Sent when a scheduled blob storage export fails.",
},
+ {
+ value: "posthog-export-failed",
+ title: "PostHog export failed",
+ description:
+ "Sent when a PostHog export is disabled after a configuration error, such as an unreachable host.",
+ },
{
value: "evaluator-blocked",
title: "Evaluator deactivated",
diff --git a/worker/src/__tests__/posthogIntegrationProjectJob.test.ts b/worker/src/__tests__/posthogIntegrationProjectJob.test.ts
index b557dafb7c..e325984abb 100644
--- a/worker/src/__tests__/posthogIntegrationProjectJob.test.ts
+++ b/worker/src/__tests__/posthogIntegrationProjectJob.test.ts
@@ -51,12 +51,72 @@ const h = vi.hoisted(() => {
})();
}
- const posthogIntegrationUpdate = vi.fn();
+ // How many rows the atomic disable claim matches. 0 simulates a lost claim:
+ // either a concurrent run already flipped enabled, or the customer corrected
+ // the hostname mid-run so the run's `where` predicate no longer matches.
+ const disableClaim = { matchedRows: 1 };
+
+ // Observable completion of the terminal notification. `settled` flips only
+ // after a macrotask, so a caller that fires the dispatch without awaiting it
+ // returns while `settled` is still false — that gap is what distinguishes an
+ // awaited dispatch from a fire-and-forget one.
+ const notification = { settled: false, rejects: false };
+
+ // Stand-in for the Prisma namespace class. The record-not-found predicate is
+ // instanceof-based, so a duck-typed `{ code: "P2025" }` would not satisfy it
+ // — tests must throw a real instance of this class.
+ class PrismaClientKnownRequestError extends Error {
+ code: string;
+ constructor(message: string, { code }: { code: string }) {
+ super(message);
+ this.name = "PrismaClientKnownRequestError";
+ this.code = code;
+ }
+ }
+
+ // Lets a test fail the lastError/lastErrorAt persist write without disturbing
+ // the tests that need it to resolve.
+ const persist = { failWith: undefined as unknown };
+
+ const posthogIntegrationUpdate = vi.fn(async () => {
+ if (persist.failWith !== undefined) throw persist.failWith;
+ });
+ // The atomic disable (enabled true->false) uses updateMany. Only the disable
+ // write is claim-controlled; any other updateMany keeps matching.
+ const posthogIntegrationUpdateMany = vi.fn(
+ async (args?: { data?: Record<string, unknown> }) => ({
+ count: args?.data?.enabled === false ? disableClaim.matchedRows : 1,
+ }),
+ );
+ // Hostname preflight (throws OutboundUrlValidationError on a bad
+ // hostname) and the customer notification, both controllable per test.
+ const validateWebhookURL = vi.fn(async () => {});
+ const recordIncrement = vi.fn();
+ const dispatchProjectNotification = vi.fn(async () => {
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ try {
+ if (notification.rejects) throw new Error("notification dispatch failed");
+ } finally {
+ notification.settled = true;
+ }
+ });
const getTraces = vi.fn(() => fakeStream("traces"));
const getGenerations = vi.fn(() => fakeStream("generations"));
const getScores = vi.fn(() => fakeStream("scores"));
const getEvents = vi.fn(() => fakeStream("events"));
+ // Minimal stand-in for the shared class. The classifier duck-types on
+ // `.name === "OutboundUrlValidationError"` + `.code` (it does not use
+ // instanceof), so this shape is sufficient for the mocked server barrel.
+ class OutboundUrlValidationError extends Error {
+ code: string;
+ constructor(code: string, message: string) {
+ super(message);
+ this.name = "OutboundUrlValidationError";
+ this.code = code;
+ }
+ }
+
const defaultIntegration = () => ({
projectId: "project-1",
enabled: true,
@@ -76,6 +136,15 @@ const h = vi.hoisted(() => {
constructed,
state,
posthogIntegrationUpdate,
+ posthogIntegrationUpdateMany,
+ PrismaClientKnownRequestError,
+ persist,
+ disableClaim,
+ notification,
+ validateWebhookURL,
+ recordIncrement,
+ dispatchProjectNotification,
+ OutboundUrlValidationError,
getTraces,
getGenerations,
getScores,
@@ -90,16 +159,25 @@ vi.mock("@langfuse/shared/src/db", () => ({
posthogIntegration: {
findFirst: vi.fn(async () => h.db.integration),
update: h.posthogIntegrationUpdate,
+ updateMany: h.posthogIntegrationUpdateMany,
},
},
+ // The record-not-found predicate lives in its own module but imports Prisma
+ // from this specifier, so this factory intercepts it too. Without this
+ // export the predicate throws a TypeError instead of returning a boolean.
+ Prisma: {
+ PrismaClientKnownRequestError: h.PrismaClientKnownRequestError,
+ },
}));
vi.mock("@langfuse/shared/src/server", () => ({
QueueName: { PostHogIntegrationProcessingQueue: "posthog" },
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
- recordIncrement: vi.fn(),
+ recordIncrement: h.recordIncrement,
getCurrentSpan: vi.fn(() => undefined),
- validateWebhookURL: vi.fn(async () => {}),
+ validateWebhookURL: h.validateWebhookURL,
+ dispatchProjectNotification: h.dispatchProjectNotification,
+ OutboundUrlValidationError: h.OutboundUrlValidationError,
getTracesForAnalyticsIntegrations: h.getTraces,
getGenerationsForAnalyticsIntegrations: h.getGenerations,
getScoresForAnalyticsIntegrations: h.getScores,
@@ -191,6 +269,17 @@ function resetSharedState() {
h.state.rowCounts.generations = 2;
h.state.rowCounts.events = 2;
h.posthogIntegrationUpdate.mockClear();
+ h.persist.failWith = undefined;
+ // mockClear, not mockReset: the implementations above are claim/notification
+ // aware and must survive between tests; only the recorded calls reset.
+ h.posthogIntegrationUpdateMany.mockClear();
+ h.disableClaim.matchedRows = 1;
+ h.notification.settled = false;
+ h.notification.rejects = false;
+ h.recordIncrement.mockClear();
+ h.validateWebhookURL.mockReset();
+ h.validateWebhookURL.mockImplementation(async () => {});
+ h.dispatchProjectNotification.mockClear();
h.getTraces.mockClear();
h.getGenerations.mockClear();
h.getScores.mockClear();
@@ -425,3 +514,282 @@ describe("handlePostHogIntegrationProjectJob delivery controls (issue #12786)",
expect(h.posthogIntegrationUpdate).not.toHaveBeenCalled();
});
});
+
+// A deterministic customer-config fault (bad/malicious hostname
+// rejected by validateWebhookURL) must be classified as the customer's problem
+// — the job RESOLVES (not thrown, so the type:failed counter never increments
+// and the Failures monitor stays quiet), the integration is auto-disabled,
+// lastError/lastErrorAt are persisted, and the customer is notified once with
+// disabled=true. Transient/infra faults still throw, retry, and trip the
+// monitor unchanged.
+describe("handlePostHogIntegrationProjectJob customer-fault auto-disable", () => {
+ beforeEach(() => {
+ resetSharedState();
+ // A known-good source + write-mode combo so the pre-export write-mode
+ // guards pass and the handler reaches the hostname preflight.
+ h.db.integration = {
+ ...h.defaultIntegration(),
+ exportSource: "TRACES_OBSERVATIONS",
+ };
+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";
+ });
+
+ // Every `data` payload written across update + updateMany, so assertions
+ // don't over-constrain which Prisma method carries which column.
+ const writeData = (): Array<Record<string, unknown>> =>
+ [
+ ...h.posthogIntegrationUpdate.mock.calls,
+ ...h.posthogIntegrationUpdateMany.mock.calls,
+ ].map(
+ (call) =>
+ (call[0] as { data?: Record<string, unknown> } | undefined)?.data ?? {},
+ );
+
+ const disableNotifications = (): number =>
+ h.dispatchProjectNotification.mock.calls.filter((call) => {
+ const arg = call[0] as { event?: { disabled?: unknown } } | undefined;
+ return arg?.event?.disabled === true;
+ }).length;
+
+ // The atomic disable writes: updateMany calls carrying `enabled: false`.
+ const disableWrites = (): Array<{
+ where?: Record<string, unknown>;
+ data?: Record<string, unknown>;
+ }> =>
+ h.posthogIntegrationUpdateMany.mock.calls
+ .map((call) => call[0] as { where?: any; data?: any } | undefined)
+ .filter((arg): arg is { where?: any; data?: any } => Boolean(arg))
+ .filter((arg) => arg.data?.enabled === false);
+
+ // Counter emitted when an integration is auto-disabled. Matched on the
+ // metric-name fragment rather than the full string so the assertion tracks
+ // the behavior, not the namespace. The winning-claim test below asserts this
+ // filter yields exactly one call, which keeps the "not recorded" assertions
+ // from passing vacuously if the metric were ever renamed away.
+ const disableMetrics = (): unknown[][] =>
+ h.recordIncrement.mock.calls.filter(
+ ([name]) =>
+ typeof name === "string" && name.includes("integration_disabled"),
+ );
+
+ const badHostnameFault = () =>
+ new h.OutboundUrlValidationError(
+ "blocked-hostname",
+ "Blocked hostname detected",
+ );
+
+ // Load-bearing: the core journey. A blocked-hostname fault must resolve (no
+ // throw) AND flip enabled to false AND persist the error AND notify once.
+ it("resolves and auto-disables on a blocked-hostname validation fault", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(
+ new h.OutboundUrlValidationError(
+ "blocked-hostname",
+ "Blocked hostname detected",
+ ),
+ );
+
+ // Must not throw — a throw would increment type:failed and fire the monitor.
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(writeData().some((data) => data.enabled === false)).toBe(true);
+ expect(
+ writeData().some(
+ (data) =>
+ typeof data.lastError === "string" &&
+ (data.lastError as string).length > 0,
+ ),
+ ).toBe(true);
+ expect(writeData().some((data) => data.lastErrorAt instanceof Date)).toBe(
+ true,
+ );
+ expect(disableNotifications()).toBe(1);
+ });
+
+ // Load-bearing: the throw site rewraps into `new Error(msg, { cause })`, so
+ // the handler must classify off the cause chain — a wrapped validation error
+ // still disables.
+ it("classifies through a wrapped cause chain and still disables", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(
+ new Error("posthog export failed", {
+ cause: new h.OutboundUrlValidationError(
+ "blocked-hostname",
+ "Blocked hostname detected",
+ ),
+ }),
+ );
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(writeData().some((data) => data.enabled === false)).toBe(true);
+ expect(disableNotifications()).toBe(1);
+ });
+
+ // A disallowed protocol or port is as deterministic as a blocked hostname —
+ // no retry can fix `ftp://` or `:8080`. These reach the handler as coded
+ // errors only because validateWebhookURL raises OutboundUrlValidationError
+ // for them; while it threw a bare Error, they were unclassifiable and
+ // retried forever. Real coded-vs-uncoded coverage lives in
+ // webhook-validation.test.ts, since this suite mocks the validator.
+ it.each([
+ ["protocol-not-allowed", "Only HTTP and HTTPS protocols are allowed"],
+ ["port-not-allowed", "Only ports 80 and 443 are allowed"],
+ ] as const)("resolves and auto-disables on %s", async (code, message) => {
+ h.validateWebhookURL.mockRejectedValueOnce(
+ new h.OutboundUrlValidationError(code, message),
+ );
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(writeData().some((data) => data.enabled === false)).toBe(true);
+ expect(disableNotifications()).toBe(1);
+ });
+
+ // Load-bearing negative controls: transient/infra faults are NOT customer
+ // faults — the handler must rethrow (so BullMQ retries and the monitor
+ // fires), leave enabled unchanged, and never notify.
+ it.each([
+ [
+ "a dns-lookup-failed validation fault",
+ () =>
+ new h.OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed for host.example",
+ ),
+ ],
+ [
+ "a generic ClickHouse-style error",
+ () => new Error("ClickHouse read failed"),
+ ],
+ ] as const)(
+ "rethrows on %s, leaves enabled unchanged, and does not notify",
+ async (_label, makeError) => {
+ h.validateWebhookURL.mockRejectedValueOnce(makeError());
+
+ await expect(
+ handlePostHogIntegrationProjectJob(makeJob()),
+ ).rejects.toThrow();
+
+ expect(writeData().some((data) => data.enabled === false)).toBe(false);
+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();
+ },
+ );
+
+ // Cheap addition (Semantics table success row): a healthy run clears any
+ // prior error state so a fixed + re-enabled integration reads as healthy.
+ it("clears lastError/lastErrorAt on a successful run", async () => {
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(h.posthogIntegrationUpdate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({ lastError: null, lastErrorAt: null }),
+ }),
+ );
+ });
+
+ // Load-bearing: the disable predicate must pin the hostname this run actually
+ // validated. Without it, a run carrying stale config would disable a hostname
+ // the customer already corrected mid-run. Asserted on the real argument, so a
+ // predicate that silently drops the host guard goes red.
+ it("scopes the atomic disable to the hostname this run validated", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(disableWrites()).toHaveLength(1);
+ expect(disableWrites()[0].where).toEqual({
+ projectId: "project-1",
+ enabled: true,
+ posthogHostName: "https://us.posthog.com",
+ });
+ });
+
+ // Positive control for the two "not recorded / not notified" assertions
+ // below: proves the disable-metric filter and the notification counter both
+ // observe something real on the winning path.
+ it("records the disable metric and notifies once when the claim is won", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(disableMetrics()).toHaveLength(1);
+ expect(disableNotifications()).toBe(1);
+ });
+
+ // Load-bearing: the notify-once guarantee under a lost claim. count === 0
+ // means either a concurrent run already flipped enabled or the customer
+ // corrected the hostname mid-run; in both cases this run must stay silent
+ // rather than claim a state it did not write.
+ it("stays silent when the disable claim matches no row", async () => {
+ h.disableClaim.matchedRows = 0;
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+
+ // Still a deliberate resolve: a lost claim is not a job failure.
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(disableWrites()).toHaveLength(1);
+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();
+ expect(disableMetrics()).toHaveLength(0);
+ });
+
+ // Load-bearing: fail-fast resolves the job and the scheduler only enqueues
+ // enabled integrations, so a dispatch left in flight has no retry path and
+ // would leave the customer silently disabled. The notification mock settles
+ // one macrotask late, so this flag is still false if the dispatch is fired
+ // without being awaited.
+ it("completes the disable notification before the handler resolves", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(h.notification.settled).toBe(true);
+ });
+
+ // Load-bearing: awaiting the dispatch must not turn the deliberate resolve
+ // into a job failure — the notify helper swallows its own errors, and the
+ // disable stands regardless of whether the customer could be reached.
+ it("resolves and keeps the integration disabled when the notification rejects", async () => {
+ h.notification.rejects = true;
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(disableWrites()).toHaveLength(1);
+ expect(h.notification.settled).toBe(true);
+ });
+
+ // Load-bearing: the integration can be deleted while the run is in flight, so
+ // the persist races the delete and throws P2025. There is nothing to persist
+ // and nobody to notify — the job is obsolete, so it must resolve quietly
+ // rather than disable a row that no longer exists or fail the job.
+ it("drops the obsolete job when the integration is deleted mid-run", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+ h.persist.failWith = new h.PrismaClientKnownRequestError(
+ "An operation failed because it depends on one or more records that were required but not found.",
+ { code: "P2025" },
+ );
+
+ await handlePostHogIntegrationProjectJob(makeJob());
+
+ expect(h.posthogIntegrationUpdate).toHaveBeenCalledTimes(1);
+ expect(disableWrites()).toHaveLength(0);
+ expect(disableMetrics()).toHaveLength(0);
+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();
+ });
+
+ // Load-bearing contrast to the case above: any other persist failure means we
+ // do NOT know the fault was recorded, so the run must not claim it as handled
+ // — rethrow and let BullMQ retry and reclassify. Without this, a swallowed DB
+ // outage would silently drop the customer fault.
+ it("rethrows when persisting the error fails for any other reason", async () => {
+ h.validateWebhookURL.mockRejectedValueOnce(badHostnameFault());
+ h.persist.failWith = new Error("database unavailable");
+
+ await expect(
+ handlePostHogIntegrationProjectJob(makeJob()),
+ ).rejects.toThrow();
+
+ expect(disableWrites()).toHaveLength(0);
+ expect(disableMetrics()).toHaveLength(0);
+ expect(h.dispatchProjectNotification).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/__tests__/webhook-validation.test.ts b/worker/src/__tests__/webhook-validation.test.ts
index e3b14ab501..0d99a20f53 100644
--- a/worker/src/__tests__/webhook-validation.test.ts
+++ b/worker/src/__tests__/webhook-validation.test.ts
@@ -88,6 +88,22 @@ describe("Webhook URL Validation", () => {
).rejects.toThrow("Only ports 80 and 443 are allowed");
});
+ // Protocol and port rejections must carry an OutboundUrlValidationError
+ // code, not just a message: callers classify deterministic config faults
+ // off the code, so an uncoded rejection is silently unclassifiable and
+ // retries forever instead of disabling the integration.
+ it.each([
+ ["ftp://example.com", "protocol-not-allowed"],
+ ["file:///etc/passwd", "protocol-not-allowed"],
+ ["https://example.com:8080/hook", "port-not-allowed"],
+ ["http://example.com:3000/hook", "port-not-allowed"],
+ ])("should reject %s with a coded error", async (url, code) => {
+ await expect(validateWebhookURL(url)).rejects.toMatchObject({
+ name: "OutboundUrlValidationError",
+ code,
+ });
+ });
+
it("should allow standard ports", async () => {
await expect(
validateWebhookURL("https://httpbin.org:443/post"),
diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.ts
index 6de62a4ec1..9b09a2aead 100644
--- a/worker/src/features/integrations/customerFaultClassification.test.ts
+++ b/worker/src/features/integrations/customerFaultClassification.test.ts
@@ -56,6 +56,7 @@ const OUTBOUND_URL_FAULTS = [
["invalid-encoding", "invalid_endpoint_url"],
["https-required", "invalid_endpoint_url"],
["protocol-not-allowed", "invalid_endpoint_url"],
+ ["port-not-allowed", "invalid_endpoint_url"],
["url-credentials-not-allowed", "invalid_endpoint_url"],
] as const;
diff --git a/worker/src/features/integrations/customerFaultClassification.ts b/worker/src/features/integrations/customerFaultClassification.ts
index e24f79c85b..216752187b 100644
--- a/worker/src/features/integrations/customerFaultClassification.ts
+++ b/worker/src/features/integrations/customerFaultClassification.ts
@@ -63,6 +63,7 @@ const INVALID_URL_OUTBOUND_URL_CODES = new Set<string>([
"invalid-encoding",
"https-required",
"protocol-not-allowed",
+ "port-not-allowed",
"url-credentials-not-allowed",
]);
diff --git a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
index 9ef9f637b5..d6777da978 100644
--- a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
+++ b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
@@ -10,6 +10,8 @@ import {
getEventsForAnalyticsIntegrations,
getCurrentSpan,
validateWebhookURL,
+ recordIncrement,
+ dispatchProjectNotification,
} from "@langfuse/shared/src/server";
import {
transformTraceForPostHog,
@@ -21,8 +23,17 @@ import { decrypt } from "@langfuse/shared/encryption";
import { PostHog } from "posthog-node";
import { recordExportVolume } from "../../services/exportVolumeMetric";
import { assertExportSourceWritable } from "../exportWriteModeGuard";
+import { classifyCustomerFault } from "../integrations/customerFaultClassification";
+import { isRecordNotFoundError } from "../integrations/prismaErrors";
import { env, v4WritesToLegacyTables } from "../../env";
+// Counter for PostHog integrations auto-disabled after a deterministic
+// customer-config fault, tagged by `reason`. A classifier-regression
+// mass-disable shows up as a spike in the non-SSRF buckets vs a near-zero
+// baseline, separate from expected SSRF/abuse disables (mirrors blob's metric).
+export const POSTHOG_INTEGRATION_DISABLED_METRIC =
+ "langfuse.posthog.integration_disabled.count";
+
type PostHogExecutionConfig = {
projectId: string;
projectName: string;
@@ -258,65 +269,70 @@ export const handlePostHogIntegrationProjectJob = async (
return;
}
- // Validate PostHog hostname to prevent SSRF attacks before sending data
try {
- await validateWebhookURL(postHogIntegration.posthogHostName);
- } catch (error) {
- logger.error(
- `[POSTHOG] PostHog integration for project ${projectId} has invalid hostname: ${postHogIntegration.posthogHostName}. Error: ${error instanceof Error ? error.message : String(error)}`,
+ // Validate PostHog hostname to prevent SSRF attacks before sending data.
+ // Rewrap preserving { cause } so the single catch below can classify the
+ // OutboundUrlValidationError off the chain.
+ try {
+ await validateWebhookURL(postHogIntegration.posthogHostName);
+ } catch (error) {
+ logger.error(
+ `[POSTHOG] PostHog integration for project ${projectId} has invalid hostname: ${postHogIntegration.posthogHostName}. Error: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ throw new Error(
+ `Invalid PostHog hostname for project ${projectId}: ${error instanceof Error ? error.message : "Unknown error"}`,
+ { cause: error },
+ );
+ }
+
+ // Resume from lastSyncAt. On first run, fall back to the project's
+ // createdAt since no trace data can precede it.
+ const minTimestamp =
+ postHogIntegration.lastSyncAt || postHogIntegration.project.createdAt;
+ const uncappedMaxTimestamp = new Date(Date.now() - 30 * 60 * 1000); // 30 minutes ago
+
+ // Cap maxTimestamp at the next UTC day boundary after minTimestamp. Bounds
+ // per-run work so a stuck integration (or a backfill on an older project)
+ // does not re-scan an ever-growing window on each hourly retry, and aligns
+ // with the toDate(...) ClickHouse partition/ordering keys for better
+ // pruning. Healthy integrations are unaffected because uncappedMaxTimestamp
+ // wins whenever the sync is within one day of present.
+ const nextDayBoundary = new Date(
+ Date.UTC(
+ minTimestamp.getUTCFullYear(),
+ minTimestamp.getUTCMonth(),
+ minTimestamp.getUTCDate() + 1,
+ ),
);
- throw new Error(
- `Invalid PostHog hostname for project ${projectId}: ${error instanceof Error ? error.message : "Unknown error"}`,
+ const maxTimestamp = new Date(
+ Math.min(nextDayBoundary.getTime(), uncappedMaxTimestamp.getTime()),
);
- }
- // Resume from lastSyncAt. On first run, fall back to the project's
- // createdAt since no trace data can precede it.
- const minTimestamp =
- postHogIntegration.lastSyncAt || postHogIntegration.project.createdAt;
- const uncappedMaxTimestamp = new Date(Date.now() - 30 * 60 * 1000); // 30 minutes ago
-
- // Cap maxTimestamp at the next UTC day boundary after minTimestamp. Bounds
- // per-run work so a stuck integration (or a backfill on an older project)
- // does not re-scan an ever-growing window on each hourly retry, and aligns
- // with the toDate(...) ClickHouse partition/ordering keys for better
- // pruning. Healthy integrations are unaffected because uncappedMaxTimestamp
- // wins whenever the sync is within one day of present.
- const nextDayBoundary = new Date(
- Date.UTC(
- minTimestamp.getUTCFullYear(),
- minTimestamp.getUTCMonth(),
- minTimestamp.getUTCDate() + 1,
- ),
- );
- const maxTimestamp = new Date(
- Math.min(nextDayBoundary.getTime(), uncappedMaxTimestamp.getTime()),
- );
+ if (maxTimestamp <= minTimestamp) {
+ logger.info(
+ `[POSTHOG] Skipping PostHog integration for project ${projectId}: empty sync window (min: ${minTimestamp.toISOString()}, max: ${maxTimestamp.toISOString()})`,
+ );
+ return;
+ }
- if (maxTimestamp <= minTimestamp) {
logger.info(
- `[POSTHOG] Skipping PostHog integration for project ${projectId}: empty sync window (min: ${minTimestamp.toISOString()}, max: ${maxTimestamp.toISOString()})`,
+ `[POSTHOG] Syncing project ${projectId} from ${minTimestamp.toISOString()} to ${maxTimestamp.toISOString()}`,
);
- return;
- }
- logger.info(
- `[POSTHOG] Syncing project ${projectId} from ${minTimestamp.toISOString()} to ${maxTimestamp.toISOString()}`,
- );
-
- // Fetch relevant data and send it to PostHog
- const executionConfig: PostHogExecutionConfig = {
- projectId,
- projectName: postHogIntegration.project.name,
- minTimestamp,
- maxTimestamp,
- decryptedPostHogApiKey: decrypt(postHogIntegration.encryptedPosthogApiKey),
- postHogHost: postHogIntegration.posthogHostName,
- useGraceHash: job.attemptsMade > 0,
- volume: { bytes: 0 },
- };
+ // Fetch relevant data and send it to PostHog
+ const executionConfig: PostHogExecutionConfig = {
+ projectId,
+ projectName: postHogIntegration.project.name,
+ minTimestamp,
+ maxTimestamp,
+ decryptedPostHogApiKey: decrypt(
+ postHogIntegration.encryptedPosthogApiKey,
+ ),
+ postHogHost: postHogIntegration.posthogHostName,
+ useGraceHash: job.attemptsMade > 0,
+ volume: { bytes: 0 },
+ };
- try {
// Fail loudly before exporting empty data and advancing lastSyncAt
// (LFE-10148, LFE-11009); the catch below logs and BullMQ retries.
assertExportSourceWritable(
@@ -376,12 +392,16 @@ export const handlePostHogIntegrationProjectJob = async (
throwIfPostHogSendError(executionConfig);
// Update the last run information for the postHogIntegration record.
+ // Clear any prior error state so a fixed + re-enabled integration reads as
+ // healthy.
await prisma.posthogIntegration.update({
where: {
projectId,
},
data: {
lastSyncAt: executionConfig.maxTimestamp,
+ lastError: null,
+ lastErrorAt: null,
},
});
// Record gzipped on-wire export volume once the run has succeeded.
@@ -393,11 +413,128 @@ export const handlePostHogIntegrationProjectJob = async (
logger.info(
`[POSTHOG] PostHog integration processing complete for project ${projectId}`,
);
+ } catch (error) {
+ // A deterministic customer-config fault (a bad/malicious hostname rejected
+ // by validateWebhookURL) can't succeed until the customer fixes it.
+ // Classify it, disable the integration, persist the error, notify once,
+ // and RESOLVE (return) — a throw would increment the BullMQ `type:failed`
+ // metric on every attempt and keep the Failures monitor lit for a fault
+ // that will never clear on its own. Transient/infra faults stay on the
+ // retry+alert path unchanged.
+ const reason = classifyCustomerFault(error);
+ // Bounded like blob's extractStorageErrorMessage: today's classified
+ // messages are short static strings, but persisted error text should not
+ // grow unbounded if a future classified path carries a large SDK body.
+ const message = (
+ error instanceof Error ? error.message : String(error)
+ ).slice(0, 1000);
+
+ if (reason === undefined) {
+ logger.error(
+ `[POSTHOG] Error processing PostHog integration for project ${projectId}`,
+ error,
+ );
+ throw error;
+ }
+
+ try {
+ await prisma.posthogIntegration.update({
+ where: { projectId },
+ data: { lastError: message, lastErrorAt: new Date() },
+ });
+ } catch (persistError) {
+ if (isRecordNotFoundError(persistError)) {
+ // Integration deleted mid-run: nothing to persist, nobody to notify.
+ // Drop the obsolete job instead of retrying it.
+ logger.info(
+ `[POSTHOG] PostHog integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${message}`,
+ );
+ return;
+ }
+ // Persisting failed for an infra reason (e.g. DB unavailable). Don't claim
+ // the customer fault as handled — rethrow so BullMQ retries and reclassifies.
+ logger.error(
+ `[POSTHOG] Failed to persist PostHog integration error for project ${projectId}`,
+ persistError,
+ );
+ throw persistError;
+ }
+
+ // Atomic disable: count === 1 means THIS worker flipped enabled true→false,
+ // so the terminal notification fires exactly once even if a concurrent run
+ // (distinct jobId, not queue-deduped) races the same fault. The host is
+ // part of the predicate so a run carrying stale config cannot disable a
+ // hostname the customer corrected mid-run — the fault we caught refers to a
+ // config that no longer exists, and the next scheduled run re-evaluates.
+ const { count } = await prisma.posthogIntegration.updateMany({
+ where: {
+ projectId,
+ enabled: true,
+ posthogHostName: postHogIntegration.posthogHostName,
+ },
+ data: { enabled: false },
+ });
+ if (count === 0) {
+ // Already disabled by a concurrent run, or the host changed under us.
+ logger.info(
+ `[POSTHOG] Skipped disabling PostHog integration for project ${projectId}: already disabled or host changed mid-run`,
+ );
+ return;
+ }
+
+ recordIncrement(POSTHOG_INTEGRATION_DISABLED_METRIC, 1, { reason });
+ logger.warn(
+ `[POSTHOG] Disabled PostHog integration for project ${projectId} after a customer fault (reason=${reason}): ${message}`,
+ { posthogDisableReason: reason, projectId },
+ );
+ // Awaited, not fire-and-forget: the job resolves immediately after this and
+ // the integration is now disabled, so the scheduler (which only enqueues
+ // enabled integrations) never revisits it. An interrupted dispatch would
+ // leave the customer silently disabled with no notification and no retry.
+ await notifyPostHogExportFailed(
+ projectId,
+ postHogIntegration.project.name,
+ postHogIntegration.posthogHostName,
+ );
+ return;
+ }
+};
+
+// Terminal "export disabled" notification. Called at most once per broken
+// integration, guarded by the atomic enabled true→false claim, so it needs no
+// cooldown of its own (matching blob's disabled path, which also bypasses it).
+// Swallows every failure: notification trouble must not turn the deliberate
+// RESOLVE back into a job failure. projectName/host are passed in rather than
+// re-read — the caller already holds them.
+async function notifyPostHogExportFailed(
+ projectId: string,
+ projectName: string,
+ host: string,
+): Promise<void> {
+ try {
+ const settingsPath = `/project/${projectId}/settings/integrations/posthog`;
+ await dispatchProjectNotification({
+ projectId,
+ event: {
+ eventType: "posthog-export-failed",
+ severity: "ALERT",
+ projectId,
+ projectName,
+ // The integration is keyed by projectId (1:1); the host is the most
+ // useful human label for the failing export destination.
+ resourceId: projectId,
+ resourceName: host || "PostHog integration",
+ message: `PostHog export disabled for project "${projectName}" after a configuration fault.`,
+ url: env.NEXTAUTH_URL
+ ? `${env.NEXTAUTH_URL}${settingsPath}`
+ : undefined,
+ disabled: true,
+ },
+ });
} catch (error) {
logger.error(
- `[POSTHOG] Error processing PostHog integration for project ${projectId}`,
+ `[POSTHOG] Failed to send failure notification for project ${projectId}`,
error,
);
- throw error;
}
-};
+}