🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, it, expect } from "vitest";
import {
transformTraceForMixpanel,
transformGenerationForMixpanel,
transformScoreForMixpanel,
transformEventForMixpanel,
} from "../features/mixpanel/transformers";
import type {
AnalyticsTraceEvent,
AnalyticsGenerationEvent,
AnalyticsScoreEvent,
AnalyticsObservationEvent,
} from "@langfuse/shared/src/server";
describe("Mixpanel transformers", () => {
const projectId = "test-project-id";
describe("transformEventForMixpanel", () => {
it("should transform an event with user_id", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-123",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "test-event",
langfuse_trace_name: "test-trace",
langfuse_trace_id: "trace-456",
langfuse_url:
"https://langfuse.com/project/test/traces/trace-456?observation=event-123",
langfuse_user_url: "https://langfuse.com/project/test/users/user-789",
langfuse_cost_usd: 0.001,
langfuse_input_units: 100,
langfuse_output_units: 50,
langfuse_total_units: 150,
langfuse_session_id: "session-abc",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-789",
langfuse_latency: 1.5,
langfuse_time_to_first_token: 0.3,
langfuse_release: "v1.0.0",
langfuse_version: "1",
langfuse_model: "gpt-4",
langfuse_level: "DEFAULT",
langfuse_type: "GENERATION",
langfuse_tags: ["tag1", "tag2"],
langfuse_environment: "production",
langfuse_event_version: "1.0.0",
posthog_session_id: "posthog-session-123",
mixpanel_session_id: "mixpanel-session-456",
};
const result = transformEventForMixpanel(event, projectId);
expect(result.event).toBe("[Langfuse] Observation");
expect(result.properties.distinct_id).toBe("user-789");
expect(result.properties.$user_id).toBe("user-789");
expect(result.properties.time).toBe(
new Date("2024-01-15T10:00:00Z").getTime(),
);
expect(result.properties.$insert_id).toBeDefined();
expect(result.properties.session_id).toBe("mixpanel-session-456");
expect(result.properties.langfuse_observation_name).toBe("test-event");
expect(result.properties.langfuse_trace_name).toBe("test-trace");
expect(result.properties.langfuse_model).toBe("gpt-4");
expect(result.properties.langfuse_type).toBe("GENERATION");
// Should not include posthog_session_id or mixpanel_session_id in properties
expect(result.properties.posthog_session_id).toBeUndefined();
expect(result.properties.mixpanel_session_id).toBeUndefined();
});
it("should transform an anonymous event without user_id", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-anonymous",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "anonymous-event",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: null,
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformEventForMixpanel(event, projectId);
expect(result.event).toBe("[Langfuse] Observation");
// distinct_id should be empty string for non-user events (Mixpanel distributes across shards)
expect(result.properties.distinct_id).toBe("");
// Should not have $user_id for anonymous events
expect(result.properties.$user_id).toBeUndefined();
expect(result.properties.session_id).toBeUndefined();
});
it("should generate consistent insert IDs for the same event", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-consistent",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "consistent-event",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: null,
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result1 = transformEventForMixpanel(event, projectId);
const result2 = transformEventForMixpanel(event, projectId);
expect(result1.properties.$insert_id).toBe(result2.properties.$insert_id);
});
it("should use langfuse_session_id when mixpanel_session_id is not available", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-with-langfuse-session",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "session-event",
langfuse_session_id: "langfuse-session-123",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-456",
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformEventForMixpanel(event, projectId);
expect(result.properties.session_id).toBe("langfuse-session-123");
});
it("should prefer mixpanel_session_id over langfuse_session_id", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-with-both-sessions",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "session-event",
langfuse_session_id: "langfuse-session-123",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-456",
langfuse_event_version: "1.0.0",
posthog_session_id: "posthog-session-789",
mixpanel_session_id: "mixpanel-session-456",
};
const result = transformEventForMixpanel(event, projectId);
expect(result.properties.session_id).toBe("mixpanel-session-456");
});
it("should include langfuse_project_name in properties", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-with-project-name",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "test-event",
langfuse_project_id: projectId,
langfuse_project_name: "My Custom Project Name",
langfuse_user_id: "user-123",
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformEventForMixpanel(event, projectId);
expect(result.properties.langfuse_project_name).toBe(
"My Custom Project Name",
);
});
});
describe("bad distinct_id handling", () => {
const badIds = [
"undefined",
"null",
"Null",
"NULL",
"0",
"-1",
"00000000-0000-0000-0000-000000000000",
"unknown",
"anonymous",
" undefined ",
"lmy47d",
];
it.each(badIds)(
"transformTraceForMixpanel falls back to empty string distinct_id when user_id is '%s'",
(badId) => {
const trace: AnalyticsTraceEvent = {
langfuse_id: "trace-bad-id",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_trace_name: "test",
langfuse_project_id: projectId,
langfuse_project_name: "Test",
langfuse_user_id: badId,
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformTraceForMixpanel(trace, projectId);
expect(result.properties.distinct_id).toBe("");
expect(result.properties.$user_id).toBeUndefined();
},
);
it.each(badIds)(
"transformGenerationForMixpanel falls back to empty string distinct_id when user_id is '%s'",
(badId) => {
const generation: AnalyticsGenerationEvent = {
langfuse_id: "gen-bad-id",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_generation_name: "test",
langfuse_trace_name: "test",
langfuse_trace_id: "trace-456",
langfuse_project_id: projectId,
langfuse_project_name: "Test",
langfuse_user_id: badId,
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformGenerationForMixpanel(generation, projectId);
expect(result.properties.distinct_id).toBe("");
expect(result.properties.$user_id).toBeUndefined();
},
);
it.each(badIds)(
"transformScoreForMixpanel falls back to empty string distinct_id when user_id is '%s'",
(badId) => {
const score: AnalyticsScoreEvent = {
langfuse_id: "score-bad-id",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_score_name: "test",
langfuse_score_value: 1,
langfuse_score_data_type: "NUMERIC",
langfuse_trace_name: "test",
langfuse_trace_id: "trace-456",
langfuse_project_id: projectId,
langfuse_project_name: "Test",
langfuse_user_id: badId,
langfuse_event_version: "1.0.0",
langfuse_score_entity_type: "trace",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformScoreForMixpanel(score, projectId);
expect(result.properties.distinct_id).toBe("");
expect(result.properties.$user_id).toBeUndefined();
},
);
it.each(badIds)(
"transformEventForMixpanel falls back to empty string distinct_id when user_id is '%s'",
(badId) => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-bad-id",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "test",
langfuse_project_id: projectId,
langfuse_project_name: "Test",
langfuse_user_id: badId,
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformEventForMixpanel(event, projectId);
expect(result.properties.distinct_id).toBe("");
expect(result.properties.$user_id).toBeUndefined();
},
);
it("should still use a valid user_id as distinct_id", () => {
const event: AnalyticsObservationEvent = {
langfuse_id: "event-valid",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_observation_name: "test",
langfuse_project_id: projectId,
langfuse_project_name: "Test",
langfuse_user_id: "real-user-123",
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: null,
};
const result = transformEventForMixpanel(event, projectId);
expect(result.properties.distinct_id).toBe("real-user-123");
expect(result.properties.$user_id).toBe("real-user-123");
});
});
describe("transformTraceForMixpanel", () => {
it("should transform a trace with user_id", () => {
const trace: AnalyticsTraceEvent = {
langfuse_id: "trace-123",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_trace_name: "test-trace",
langfuse_url: "https://langfuse.com/project/test/traces/trace-123",
langfuse_user_url: "https://langfuse.com/project/test/users/user-789",
langfuse_cost_usd: 0.01,
langfuse_count_observations: 5,
langfuse_session_id: "session-abc",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-789",
langfuse_latency: 2.5,
langfuse_release: "v1.0.0",
langfuse_version: "1",
langfuse_tags: ["tag1"],
langfuse_environment: "production",
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: "mixpanel-session-123",
};
const result = transformTraceForMixpanel(trace, projectId);
expect(result.event).toBe("[Langfuse] Trace");
expect(result.properties.distinct_id).toBe("user-789");
expect(result.properties.$user_id).toBe("user-789");
expect(result.properties.session_id).toBe("mixpanel-session-123");
});
});
describe("transformGenerationForMixpanel", () => {
it("should transform a generation with user_id", () => {
const generation: AnalyticsGenerationEvent = {
langfuse_id: "gen-123",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_generation_name: "test-generation",
langfuse_trace_name: "test-trace",
langfuse_trace_id: "trace-456",
langfuse_url:
"https://langfuse.com/project/test/traces/trace-456?observation=gen-123",
langfuse_user_url: "https://langfuse.com/project/test/users/user-789",
langfuse_cost_usd: 0.005,
langfuse_input_units: 200,
langfuse_output_units: 100,
langfuse_total_units: 300,
langfuse_session_id: "session-abc",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-789",
langfuse_latency: 1.2,
langfuse_time_to_first_token: 0.2,
langfuse_release: "v1.0.0",
langfuse_version: "1",
langfuse_model: "gpt-4-turbo",
langfuse_level: "DEFAULT",
langfuse_tags: ["api"],
langfuse_environment: "staging",
langfuse_event_version: "1.0.0",
posthog_session_id: null,
mixpanel_session_id: "mixpanel-session-456",
};
const result = transformGenerationForMixpanel(generation, projectId);
expect(result.event).toBe("[Langfuse] Generation");
expect(result.properties.distinct_id).toBe("user-789");
expect(result.properties.$user_id).toBe("user-789");
expect(result.properties.session_id).toBe("mixpanel-session-456");
expect(result.properties.langfuse_model).toBe("gpt-4-turbo");
});
});
describe("transformScoreForMixpanel", () => {
it("should transform a score with user_id", () => {
const score: AnalyticsScoreEvent = {
langfuse_id: "score-123",
timestamp: new Date("2024-01-15T10:00:00Z"),
langfuse_score_name: "quality",
langfuse_score_value: 0.95,
langfuse_score_comment: "Good response",
langfuse_score_metadata: { source: "human" },
langfuse_score_string_value: null,
langfuse_score_data_type: "NUMERIC",
langfuse_trace_name: "test-trace",
langfuse_trace_id: "trace-456",
langfuse_user_url: "https://langfuse.com/project/test/users/user-789",
langfuse_session_id: "session-abc",
langfuse_project_id: projectId,
langfuse_project_name: "Test Project",
langfuse_user_id: "user-789",
langfuse_release: "v1.0.0",
langfuse_tags: ["human-eval"],
langfuse_environment: "production",
langfuse_event_version: "1.0.0",
langfuse_score_entity_type: "trace",
langfuse_dataset_run_id: null,
posthog_session_id: null,
mixpanel_session_id: "mixpanel-session-789",
};
const result = transformScoreForMixpanel(score, projectId);
expect(result.event).toBe("[Langfuse] Score");
expect(result.properties.distinct_id).toBe("user-789");
expect(result.properties.$user_id).toBe("user-789");
expect(result.properties.session_id).toBe("mixpanel-session-789");
expect(result.properties.langfuse_score_name).toBe("quality");
expect(result.properties.langfuse_score_value).toBe(0.95);
});
});
});