🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): classify Mixpanel customer-config faults for observability (#16283)
Niklas Semmlera8a96b586629
diff --git a/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts b/worker/src/__tests__/mixpanelIntegrationProjectJob.test.tsindex dc1a6ef15d..a631f40d57 100644--- a/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts+++ b/worker/src/__tests__/mixpanelIntegrationProjectJob.test.ts@@ -43,6 +43,8 @@ const h = vi.hoisted(() => {const mixpanelIntegrationUpdate = vi.fn();+ const flushError: { value: Error | undefined } = { value: undefined };+const defaultIntegration = () => ({projectId: "project-1",enabled: true,@@ -65,6 +67,7 @@ const h = vi.hoisted(() => {mixpanelIntegrationUpdate,defaultIntegration,db,+ flushError,};});@@ -72,6 +75,11 @@ vi.mock("../features/mixpanel/mixpanelClient", () => ({MixpanelClient: class {addEvent = vi.fn();flush = vi.fn().mockImplementation(async () => {+ if (h.flushError.value) {+ const err = h.flushError.value;+ h.flushError.value = undefined; // reject once, then behave+ throw err;+ }h.timeline.push("flush");});getBatchSize = vi.fn().mockReturnValue(0);@@ -130,11 +138,26 @@ vi.mock("@langfuse/shared/src/server", () => ({}));// Import after mocks are registered.-import { handleMixpanelIntegrationProjectJob } from "../features/mixpanel/handleMixpanelIntegrationProjectJob";-import { getScoresForAnalyticsIntegrations } from "@langfuse/shared/src/server";+import {+ handleMixpanelIntegrationProjectJob,+ MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC,+} from "../features/mixpanel/handleMixpanelIntegrationProjectJob";+import {+ getScoresForAnalyticsIntegrations,+ recordIncrement,+} from "@langfuse/shared/src/server";import { env } from "../env";-function makeJob() {+// Mirrors the real queue's defaultJobOptions (mixpanelIntegrationProcessingQueue.ts).+function makeJob(attemptsMade = 0) {+ return {+ data: { id: "job-1", payload: { projectId: "project-1" } },+ attemptsMade,+ opts: { attempts: 5 },+ } as unknown as Parameters<typeof handleMixpanelIntegrationProjectJob>[0];+}++function makeJobWithoutOpts() {return {data: { id: "job-1", payload: { projectId: "project-1" } },attemptsMade: 0,@@ -267,3 +290,76 @@ describe("handleMixpanelIntegrationProjectJob legacy-mode enriched guard (LFE-11expect(h.mixpanelIntegrationUpdate).toHaveBeenCalledTimes(1);});});++describe("handleMixpanelIntegrationProjectJob customer-fault observation (no disable)", () => {+ beforeEach(() => {+ h.timeline.length = 0;+ h.mixpanelIntegrationUpdate.mockClear();+ h.db.integration = h.defaultIntegration();+ h.flushError.value = undefined;+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";+ });++ it("still throws a 401-shaped error and never calls update to disable the integration", async () => {+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");+ Object.assign(unauthorized, { statusCode: 401 });+ h.flushError.value = unauthorized;++ await expect(handleMixpanelIntegrationProjectJob(makeJob())).rejects.toBe(+ unauthorized,+ );++ expect(h.mixpanelIntegrationUpdate).not.toHaveBeenCalled();+ });++ it("a job with no opts still does not disable on a classified-fault-shaped error", async () => {+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");+ Object.assign(unauthorized, { statusCode: 401 });+ h.flushError.value = unauthorized;++ await expect(+ handleMixpanelIntegrationProjectJob(makeJobWithoutOpts()),+ ).rejects.toBe(unauthorized);++ expect(h.mixpanelIntegrationUpdate).not.toHaveBeenCalled();+ });+});++describe("handleMixpanelIntegrationProjectJob customer-fault observability", () => {+ beforeEach(() => {+ h.timeline.length = 0;+ h.mixpanelIntegrationUpdate.mockClear();+ h.db.integration = h.defaultIntegration();+ h.flushError.value = undefined;+ vi.mocked(recordIncrement).mockClear();+ (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";+ });++ it("records the customer-fault metric with reason and attempt, then still rethrows", async () => {+ const unauthorized = new Error("Mixpanel API error: 401 Unauthorized");+ Object.assign(unauthorized, { statusCode: 401 });+ h.flushError.value = unauthorized;++ await expect(handleMixpanelIntegrationProjectJob(makeJob(2))).rejects.toBe(+ unauthorized,+ );++ expect(recordIncrement).toHaveBeenCalledWith(+ MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC,+ 1,+ { reason: "credentials", attempt: 2 },+ );+ });++ it("does not record the customer-fault metric for an unclassified error", async () => {+ const infraError = new Error("connection reset");+ Object.assign(infraError, { code: "ECONNRESET" });+ h.flushError.value = infraError;++ await expect(handleMixpanelIntegrationProjectJob(makeJob())).rejects.toBe(+ infraError,+ );++ expect(recordIncrement).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.tsindex 9b09a2aead..3d31819875 100644--- a/worker/src/features/integrations/customerFaultClassification.test.ts+++ b/worker/src/features/integrations/customerFaultClassification.test.ts@@ -111,6 +111,24 @@ describe("isCustomerFaultError", () => {Object.assign(err, { $metadata: { httpStatusCode: 401 } });expect(isCustomerFaultError(wrapped(err))).toBe(true);});++ it("classifies a bare Error with no statusCode as other (control)", () => {+ const err = new Error("Unauthorized");+ expect(isCustomerFaultError(err)).toBe(false);+ expect(isCustomerFaultError(wrapped(err))).toBe(false);+ });++ it("classifies a raw .statusCode of 401 as customer_fault (unwrapped)", () => {+ const err = new Error("Unauthorized");+ Object.assign(err, { statusCode: 401 });+ expect(isCustomerFaultError(err)).toBe(true);+ });++ it("classifies a .statusCode of 401 wrapped in { cause } as customer_fault", () => {+ const err = new Error("Unauthorized");+ Object.assign(err, { statusCode: 401 });+ expect(isCustomerFaultError(wrapped(err))).toBe(true);+ });});describe("other — transient / infra / unknown (bias toward investigation)", () => {@@ -267,6 +285,13 @@ describe("classifyCustomerFault — disable reason buckets", () => {);});+ it("maps a bare .statusCode of 401 to credentials, bare and cause-wrapped", () => {+ const unauthorized = new Error("Unauthorized");+ Object.assign(unauthorized, { statusCode: 401 });+ expect(classifyCustomerFault(unauthorized)).toBe("credentials");+ expect(classifyCustomerFault(wrapped(unauthorized))).toBe("credentials");+ });+it.each([["NoSuchBucket", 404],["InvalidBucketName", 400],diff --git a/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts b/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.tsindex c95e4676d8..07948ac338 100644--- a/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts+++ b/worker/src/features/mixpanel/handleMixpanelIntegrationProjectJob.ts@@ -9,6 +9,7 @@ import {getScoresForAnalyticsIntegrations,getEventsForAnalyticsIntegrations,getCurrentSpan,+ recordIncrement,} from "@langfuse/shared/src/server";import { decrypt } from "@langfuse/shared/encryption";import { MixpanelClient } from "./mixpanelClient";@@ -21,6 +22,10 @@ import {} from "./transformers";import { env, v4WritesToLegacyTables } from "../../env";import { assertExportSourceWritable } from "../exportWriteModeGuard";+import { classifyCustomerFault } from "../integrations/customerFaultClassification";++export const MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC =+ "langfuse.mixpanel.integration_customer_fault.count";const sleep = (ms: number) =>ms > 0@@ -307,9 +312,22 @@ export const handleMixpanelIntegrationProjectJob = async (`[MIXPANEL] Mixpanel integration processing complete for project ${projectId}`,);} catch (error) {+ const mixpanelFaultReason = classifyCustomerFault(error);+ if (mixpanelFaultReason !== undefined) {+ recordIncrement(MIXPANEL_INTEGRATION_CUSTOMER_FAULT_METRIC, 1, {+ reason: mixpanelFaultReason,+ attempt: job.attemptsMade,+ });+ }logger.error(`[MIXPANEL] Error processing Mixpanel integration for project ${projectId}`,- error,+ {+ error,+ errorMessage: error instanceof Error ? error.message : String(error),+ errorStack: error instanceof Error ? error.stack : undefined,+ mixpanelFaultReason,+ attempt: job.attemptsMade,+ },);throw error;}diff --git a/worker/src/features/mixpanel/mixpanelClient.test.ts b/worker/src/features/mixpanel/mixpanelClient.test.tsnew file mode 100644index 0000000000..9e595aac37--- /dev/null+++ b/worker/src/features/mixpanel/mixpanelClient.test.ts@@ -0,0 +1,186 @@+import { describe, it, expect, vi, beforeEach } from "vitest";++const { fetchWithSecureRedirects } = vi.hoisted(() => ({+ fetchWithSecureRedirects: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },+ fetchWithSecureRedirects,+}));++// Stubbed directly rather than through the real module, which pulls in+// validateWebhookURL/whitelistFromEnv from the mocked server barrel above.+vi.mock("../analyticsIntegrationEgress", () => ({+ buildAnalyticsRedirectOptions: () => ({+ maxRedirects: 10,+ redirectValidation: { validateUrl: () => {} },+ }),+ rethrowIfOutboundValidationFailure: () => undefined,+}));++vi.mock("../../env", () => ({+ env: { LANGFUSE_MIXPANEL_TIMEOUT_MS: 5000 },+}));++import { MixpanelClient } from "./mixpanelClient";++const FAKE_PROJECT_TOKEN = "sk-mp-secret-token-abcdef123456";++const fakeResponse = (+ overrides: Partial<{+ ok: boolean;+ status: number;+ statusText: string;+ text: () => Promise<string>;+ json: () => Promise<unknown>;+ }>,+) => ({+ ok: true,+ status: 200,+ statusText: "OK",+ text: async () => "",+ json: async () => ({}),+ ...overrides,+});++const buildClient = () =>+ new MixpanelClient({ projectToken: FAKE_PROJECT_TOKEN, region: "api" });++const addOneEvent = (client: MixpanelClient) =>+ client.addEvent({+ event: "trace",+ properties: {+ time: Date.now(),+ distinct_id: "user-1",+ $insert_id: "insert-1",+ },+ });++describe("MixpanelClient.flush error handling", () => {+ beforeEach(() => {+ fetchWithSecureRedirects.mockReset();+ });++ it("throws on a 401 response with a plain, classifiable shape", async () => {+ fetchWithSecureRedirects.mockResolvedValue({+ response: fakeResponse({+ ok: false,+ status: 401,+ statusText: "Unauthorized",+ text: async () => "unauthorized",+ }),+ });++ const client = buildClient();+ addOneEvent(client);++ await expect(client.flush()).rejects.toThrow(/401/);+ });++ it("resolves (does not throw) on a 400 with partial success", async () => {+ // Mixpanel's /import can report some records imported and some rejected+ // on a 400; only a zero-imported batch is a hard failure.+ fetchWithSecureRedirects.mockResolvedValue({+ response: fakeResponse({+ ok: false,+ status: 400,+ statusText: "Bad Request",+ text: async () =>+ JSON.stringify({+ code: 400,+ num_records_imported: 1,+ failed_records: [+ { index: 1, insert_id: "x", field: "time", message: "bad" },+ ],+ }),+ }),+ });++ const client = buildClient();+ addOneEvent(client);++ await expect(client.flush()).resolves.toBeUndefined();+ });++ it("throws on a 400 where zero records were imported", async () => {+ fetchWithSecureRedirects.mockResolvedValue({+ response: fakeResponse({+ ok: false,+ status: 400,+ statusText: "Bad Request",+ text: async () =>+ JSON.stringify({ code: 400, num_records_imported: 0 }),+ }),+ });++ const client = buildClient();+ addOneEvent(client);++ await expect(client.flush()).rejects.toThrow(/400/);+ });++ it("thrown message on a 401 carries neither the project token nor the response body", async () => {+ const responseBody = "leaky-response-body-marker";+ fetchWithSecureRedirects.mockResolvedValue({+ response: fakeResponse({+ ok: false,+ status: 401,+ statusText: "Unauthorized",+ text: async () => responseBody,+ }),+ });++ const client = buildClient();+ addOneEvent(client);++ let caught: unknown;+ try {+ await client.flush();+ } catch (e) {+ caught = e;+ }+ expect(caught).toBeInstanceOf(Error);+ const message = (caught as Error).message;+ expect(message).not.toContain(FAKE_PROJECT_TOKEN);+ expect(message).not.toContain(responseBody);+ });++ it("thrown message on a fully-rejected 400 carries neither the project token nor the response body", async () => {+ const failureDetail = "very-specific-field-validation-failure-detail";+ fetchWithSecureRedirects.mockResolvedValue({+ response: fakeResponse({+ ok: false,+ status: 400,+ statusText: "Bad Request",+ text: async () =>+ JSON.stringify({+ code: 400,+ num_records_imported: 0,+ failed_records: [+ {+ index: 0,+ insert_id: "x",+ field: "time",+ message: failureDetail,+ },+ ],+ }),+ }),+ });++ const client = buildClient();+ addOneEvent(client);++ let caught: unknown;+ try {+ await client.flush();+ } catch (e) {+ caught = e;+ }+ expect(caught).toBeInstanceOf(Error);+ const message = (caught as Error).message;+ expect(message).not.toContain(FAKE_PROJECT_TOKEN);+ expect(message).not.toContain(failureDetail);+ });+});diff --git a/worker/src/features/mixpanel/mixpanelClient.ts b/worker/src/features/mixpanel/mixpanelClient.tsindex 52e1949895..69468ce80f 100644--- a/worker/src/features/mixpanel/mixpanelClient.ts+++ b/worker/src/features/mixpanel/mixpanelClient.ts@@ -147,8 +147,11 @@ export class MixpanelClient {`Failed to send events to Mixpanel: ${response.status} ${response.statusText}`,{ body: errorText },);- throw new Error(- `Mixpanel API error: ${response.status} ${response.statusText}`,+ throw Object.assign(+ new Error(+ `Mixpanel API error: ${response.status} ${response.statusText}`,+ ),+ { statusCode: response.status },);}