Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): classify Mixpanel customer-config faults for observability (#16283)

Niklas Semmlera8a96b586629
diff --git a/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts b/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts
index dc1a6ef15d..a631f40d57 100644
--- a/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts
+++ b/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts
@@ -43,6 +43,8 @@ const h = vi.hoisted(() => {
const mixpanelIntegrationUpdate = vi.fn();
+ const flushError: { value: Error | undefined } = { value: undefined };
+
const defaultIntegration = () => ({
projectId: "project-1",
enabled: true,
@@ -65,6 +67,7 @@ const h = vi.hoisted(() => {
mixpanelIntegrationUpdate,
defaultIntegration,
db,
+ flushError,
};
});
@@ -72,6 +75,11 @@ vi.mock("../features/mixpanel/mixpanelClient", () => ({
MixpanelClient: class {
addEvent = vi.fn();
flush = vi.fn().mockImplementation(async () => {
+ if (h.flushError.value) {
+ const err = h.flushError.value;
+ h.flushError.value = undefined; // reject once, then behave
+ throw err;
+ }
h.timeline.push("flush");
});
getBatchSize = vi.fn().mockReturnValue(0);
@@ -130,11 +138,26 @@ vi.mock("@langfuse/shared/src/server", () => ({
}));
// Import after mocks are registered.
-import { handleMixpanelIntegrationProjectJob } from "../features/mixpanel/handleMixpanelIntegrationProjectJob";
-import { getScoresForAnalyticsIntegrations } from "@langfuse/shared/src/server";
+import {
+ handleMixpanelIntegrationProjectJob,
+ MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC,
+} from "../features/mixpanel/handleMixpanelIntegrationProjectJob";
+import {
+ getScoresForAnalyticsIntegrations,
+ recordIncrement,
+} from "@langfuse/shared/src/server";
import { env } from "../env";
-function makeJob() {
+// Mirrors the real queue's defaultJobOptions (mixpanelIntegrationProcessingQueue.ts).
+function makeJob(attemptsMade = 0) {
+ return {
+ data: { id: "job-1", payload: { projectId: "project-1" } },
+ attemptsMade,
+ opts: { attempts: 5 },
+ } as unknown as Parameters<typeof handleMixpanelIntegrationProjectJob>[0];
+}
+
+function makeJobWithoutOpts() {
return {
data: { id: "job-1", payload: { projectId: "project-1" } },
attemptsMade: 0,
@@ -267,3 +290,76 @@ describe("handleMixpanelIntegrationProjectJob legacy-mode enriched guard (LFE-11
expect(h.mixpanelIntegrationUpdate).toHaveBeenCalledTimes(1);
});
});
+
+describe("handleMixpanelIntegrationProjectJob customer-fault observation (no disable)", () => {
+ beforeEach(() => {
+ h.timeline.length = 0;
+ h.mixpanelIntegrationUpdate.mockClear();
+ h.db.integration = h.defaultIntegration();
+ h.flushError.value = undefined;
+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";
+ });
+
+ it("still throws a 401-shaped error and never calls update to disable the integration", async () => {
+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");
+ Object.assign(unauthorized, { statusCode: 401 });
+ h.flushError.value = unauthorized;
+
+ await expect(handleMixpanelIntegrationProjectJob(makeJob())).rejects.toBe(
+ unauthorized,
+ );
+
+ expect(h.mixpanelIntegrationUpdate).not.toHaveBeenCalled();
+ });
+
+ it("a job with no opts still does not disable on a classified-fault-shaped error", async () => {
+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");
+ Object.assign(unauthorized, { statusCode: 401 });
+ h.flushError.value = unauthorized;
+
+ await expect(
+ handleMixpanelIntegrationProjectJob(makeJobWithoutOpts()),
+ ).rejects.toBe(unauthorized);
+
+ expect(h.mixpanelIntegrationUpdate).not.toHaveBeenCalled();
+ });
+});
+
+describe("handleMixpanelIntegrationProjectJob customer-fault observability", () => {
+ beforeEach(() => {
+ h.timeline.length = 0;
+ h.mixpanelIntegrationUpdate.mockClear();
+ h.db.integration = h.defaultIntegration();
+ h.flushError.value = undefined;
+ vi.mocked(recordIncrement).mockClear();
+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";
+ });
+
+ it("records the customer-fault metric with reason and attempt, then still rethrows", async () => {
+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");
+ Object.assign(unauthorized, { statusCode: 401 });
+ h.flushError.value = unauthorized;
+
+ await expect(handleMixpanelIntegrationProjectJob(makeJob(2))).rejects.toBe(
+ unauthorized,
+ );
+
+ expect(recordIncrement).toHaveBeenCalledWith(
+ MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC,
+ 1,
+ { reason: "credentials", attempt: 2 },
+ );
+ });
+
+ it("does not record the customer-fault metric for an unclassified error", async () => {
+ const infraError = new Error("connection reset");
+ Object.assign(infraError, { code: "ECONNRESET" });
+ h.flushError.value = infraError;
+
+ await expect(handleMixpanelIntegrationProjectJob(makeJob())).rejects.toBe(
+ infraError,
+ );
+
+ expect(recordIncrement).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.ts
index 9b09a2aead..3d31819875 100644
--- a/worker/src/features/integrations/customerFaultClassification.test.ts
+++ b/worker/src/features/integrations/customerFaultClassification.test.ts
@@ -111,6 +111,24 @@ describe("isCustomerFaultError", () => {
Object.assign(err, { $metadata: { httpStatusCode: 401 } });
expect(isCustomerFaultError(wrapped(err))).toBe(true);
});
+
+ it("classifies a bare Error with no statusCode as other (control)", () => {
+ const err = new Error("Unauthorized");
+ expect(isCustomerFaultError(err)).toBe(false);
+ expect(isCustomerFaultError(wrapped(err))).toBe(false);
+ });
+
+ it("classifies a raw .statusCode of 401 as customer_fault (unwrapped)", () => {
+ const err = new Error("Unauthorized");
+ Object.assign(err, { statusCode: 401 });
+ expect(isCustomerFaultError(err)).toBe(true);
+ });
+
+ it("classifies a .statusCode of 401 wrapped in { cause } as customer_fault", () => {
+ const err = new Error("Unauthorized");
+ Object.assign(err, { statusCode: 401 });
+ expect(isCustomerFaultError(wrapped(err))).toBe(true);
+ });
});
describe("other — transient / infra / unknown (bias toward investigation)", () => {
@@ -267,6 +285,13 @@ describe("classifyCustomerFault — disable reason buckets", () => {
);
});
+ it("maps a bare .statusCode of 401 to credentials, bare and cause-wrapped", () => {
+ const unauthorized = new Error("Unauthorized");
+ Object.assign(unauthorized, { statusCode: 401 });
+ expect(classifyCustomerFault(unauthorized)).toBe("credentials");
+ expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");
+ });
+
it.each([
["NoSuchBucket", 404],
["InvalidBucketName", 400],
diff --git a/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts b/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts
index c95e4676d8..07948ac338 100644
--- a/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts
+++ b/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts
@@ -9,6 +9,7 @@ import {
getScoresForAnalyticsIntegrations,
getEventsForAnalyticsIntegrations,
getCurrentSpan,
+ recordIncrement,
} from "@langfuse/shared/src/server";
import { decrypt } from "@langfuse/shared/encryption";
import { MixpanelClient } from "./mixpanelClient";
@@ -21,6 +22,10 @@ import {
} from "./transformers";
import { env, v4WritesToLegacyTables } from "../../env";
import { assertExportSourceWritable } from "../exportWriteModeGuard";
+import { classifyCustomerFault } from "../integrations/customerFaultClassification";
+
+export const MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC =
+ "langfuse.mixpanel.integration_customer_fault.count";
const sleep = (ms: number) =>
ms > 0
@@ -307,9 +312,22 @@ export const handleMixpanelIntegrationProjectJob = async (
`[MIXPANEL] Mixpanel integration processing complete for project ${projectId}`,
);
} catch (error) {
+ const mixpanelFaultReason = classifyCustomerFault(error);
+ if (mixpanelFaultReason !== undefined) {
+ recordIncrement(MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC, 1, {
+ reason: mixpanelFaultReason,
+ attempt: job.attemptsMade,
+ });
+ }
logger.error(
`[MIXPANEL] Error processing Mixpanel integration for project ${projectId}`,
- error,
+ {
+ error,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ errorStack: error instanceof Error ? error.stack : undefined,
+ mixpanelFaultReason,
+ attempt: job.attemptsMade,
+ },
);
throw error;
}
diff --git a/worker/src/features/mixpanel/mixpanelClient.test.ts b/worker/src/features/mixpanel/mixpanelClient.test.ts
new file mode 100644
index 0000000000..9e595aac37
--- /dev/null
+++ b/worker/src/features/mixpanel/mixpanelClient.test.ts
@@ -0,0 +1,186 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+const { fetchWithSecureRedirects } = vi.hoisted(() => ({
+ fetchWithSecureRedirects: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+ fetchWithSecureRedirects,
+}));
+
+// Stubbed directly rather than through the real module, which pulls in
+// validateWebhookURL/whitelistFromEnv from the mocked server barrel above.
+vi.mock("../analyticsIntegrationEgress", () => ({
+ buildAnalyticsRedirectOptions: () => ({
+ maxRedirects: 10,
+ redirectValidation: { validateUrl: () => {} },
+ }),
+ rethrowIfOutboundValidationFailure: () => undefined,
+}));
+
+vi.mock("../../env", () => ({
+ env: { LANGFUSE_MIXPANEL_TIMEOUT_MS: 5000 },
+}));
+
+import { MixpanelClient } from "./mixpanelClient";
+
+const FAKE_PROJECT_TOKEN = "sk-mp-secret-token-abcdef123456";
+
+const fakeResponse = (
+ overrides: Partial<{
+ ok: boolean;
+ status: number;
+ statusText: string;
+ text: () => Promise<string>;
+ json: () => Promise<unknown>;
+ }>,
+) => ({
+ ok: true,
+ status: 200,
+ statusText: "OK",
+ text: async () => "",
+ json: async () => ({}),
+ ...overrides,
+});
+
+const buildClient = () =>
+ new MixpanelClient({ projectToken: FAKE_PROJECT_TOKEN, region: "api" });
+
+const addOneEvent = (client: MixpanelClient) =>
+ client.addEvent({
+ event: "trace",
+ properties: {
+ time: Date.now(),
+ distinct_id: "user-1",
+ $insert_id: "insert-1",
+ },
+ });
+
+describe("MixpanelClient.flush error handling", () => {
+ beforeEach(() => {
+ fetchWithSecureRedirects.mockReset();
+ });
+
+ it("throws on a 401 response with a plain, classifiable shape", async () => {
+ fetchWithSecureRedirects.mockResolvedValue({
+ response: fakeResponse({
+ ok: false,
+ status: 401,
+ statusText: "Unauthorized",
+ text: async () => "unauthorized",
+ }),
+ });
+
+ const client = buildClient();
+ addOneEvent(client);
+
+ await expect(client.flush()).rejects.toThrow(/401/);
+ });
+
+ it("resolves (does not throw) on a 400 with partial success", async () => {
+ // Mixpanel's /import can report some records imported and some rejected
+ // on a 400; only a zero-imported batch is a hard failure.
+ fetchWithSecureRedirects.mockResolvedValue({
+ response: fakeResponse({
+ ok: false,
+ status: 400,
+ statusText: "Bad Request",
+ text: async () =>
+ JSON.stringify({
+ code: 400,
+ num_records_imported: 1,
+ failed_records: [
+ { index: 1, insert_id: "x", field: "time", message: "bad" },
+ ],
+ }),
+ }),
+ });
+
+ const client = buildClient();
+ addOneEvent(client);
+
+ await expect(client.flush()).resolves.toBeUndefined();
+ });
+
+ it("throws on a 400 where zero records were imported", async () => {
+ fetchWithSecureRedirects.mockResolvedValue({
+ response: fakeResponse({
+ ok: false,
+ status: 400,
+ statusText: "Bad Request",
+ text: async () =>
+ JSON.stringify({ code: 400, num_records_imported: 0 }),
+ }),
+ });
+
+ const client = buildClient();
+ addOneEvent(client);
+
+ await expect(client.flush()).rejects.toThrow(/400/);
+ });
+
+ it("thrown message on a 401 carries neither the project token nor the response body", async () => {
+ const responseBody = "leaky-response-body-marker";
+ fetchWithSecureRedirects.mockResolvedValue({
+ response: fakeResponse({
+ ok: false,
+ status: 401,
+ statusText: "Unauthorized",
+ text: async () => responseBody,
+ }),
+ });
+
+ const client = buildClient();
+ addOneEvent(client);
+
+ let caught: unknown;
+ try {
+ await client.flush();
+ } catch (e) {
+ caught = e;
+ }
+ expect(caught).toBeInstanceOf(Error);
+ const message = (caught as Error).message;
+ expect(message).not.toContain(FAKE_PROJECT_TOKEN);
+ expect(message).not.toContain(responseBody);
+ });
+
+ it("thrown message on a fully-rejected 400 carries neither the project token nor the response body", async () => {
+ const failureDetail = "very-specific-field-validation-failure-detail";
+ fetchWithSecureRedirects.mockResolvedValue({
+ response: fakeResponse({
+ ok: false,
+ status: 400,
+ statusText: "Bad Request",
+ text: async () =>
+ JSON.stringify({
+ code: 400,
+ num_records_imported: 0,
+ failed_records: [
+ {
+ index: 0,
+ insert_id: "x",
+ field: "time",
+ message: failureDetail,
+ },
+ ],
+ }),
+ }),
+ });
+
+ const client = buildClient();
+ addOneEvent(client);
+
+ let caught: unknown;
+ try {
+ await client.flush();
+ } catch (e) {
+ caught = e;
+ }
+ expect(caught).toBeInstanceOf(Error);
+ const message = (caught as Error).message;
+ expect(message).not.toContain(FAKE_PROJECT_TOKEN);
+ expect(message).not.toContain(failureDetail);
+ });
+});
diff --git a/worker/src/features/mixpanel/mixpanelClient.ts b/worker/src/features/mixpanel/mixpanelClient.ts
index 52e1949895..69468ce80f 100644
--- a/worker/src/features/mixpanel/mixpanelClient.ts
+++ b/worker/src/features/mixpanel/mixpanelClient.ts
@@ -147,8 +147,11 @@ export class MixpanelClient {
`Failed to send events to Mixpanel: ${response.status} ${response.statusText}`,
{ body: errorText },
);
- throw new Error(
- `Mixpanel API error: ${response.status} ${response.statusText}`,
+ throw Object.assign(
+ new Error(
+ `Mixpanel API error: ${response.status} ${response.statusText}`,
+ ),
+ { statusCode: response.status },
);
}