🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(analytics): track authenticated backend activity (#15963)
Max Deichmanncc4904f685f7
diff --git a/web/src/__tests__/server/backend-activity-middleware.servertest.ts b/web/src/__tests__/server/backend-activity-middleware.servertest.tsnew file mode 100644index 0000000000..6eb03f240c--- /dev/null+++ b/web/src/__tests__/server/backend-activity-middleware.servertest.ts@@ -0,0 +1,116 @@+import type { Session } from "next-auth";+import * as z from "zod";++const { recordBackendActivityMock } = vi.hoisted(() => ({+ recordBackendActivityMock: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("@/src/features/posthog-analytics/server/backendActivity", () => ({+ recordBackendActivity: recordBackendActivityMock,+}));++import {+ createInnerTRPCContext,+ createTRPCRouter,+ protectedOrganizationProcedure,+ protectedProjectProcedureWithoutTracing,+} from "@/src/server/api/trpc";++const activityRouter = createTRPCRouter({+ project: protectedProjectProcedureWithoutTracing+ .input(z.object({ projectId: z.string() }))+ .query(() => ({ ok: true })),+ organization: protectedOrganizationProcedure+ .input(z.object({ orgId: z.string() }))+ .query(() => ({ ok: true })),+});++const session = {+ expires: "1",+ user: {+ id: "user-1",+ email: "[email protected]",+ name: "Example User",+ admin: false,+ canCreateOrganizations: true,+ featureFlags: {},+ organizations: [+ {+ id: "org-1",+ name: "Example Organization",+ role: "MEMBER",+ plan: "cloud:hobby",+ cloudConfig: undefined,+ metadata: {},+ projects: [+ {+ id: "project-1",+ name: "Example Project",+ role: "MEMBER",+ retentionDays: 30,+ deletedAt: null,+ },+ ],+ },+ ],+ },+ environment: {},+} as Session;++const createCaller = (callerSession: Session | null = session) =>+ activityRouter.createCaller(+ createInnerTRPCContext({ session: callerSession, headers: {} }),+ );++describe("backend activity authorization middleware", () => {+ beforeEach(() => vi.clearAllMocks());++ it("records server-derived user, organization, and project ids after project authorization", async () => {+ await expect(+ createCaller().project({ projectId: "project-1" }),+ ).resolves.toEqual({ ok: true });++ expect(recordBackendActivityMock).toHaveBeenCalledOnce();+ expect(recordBackendActivityMock).toHaveBeenCalledWith({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });+ });++ it("records organization activity after organization authorization", async () => {+ await expect(+ createCaller().organization({ orgId: "org-1" }),+ ).resolves.toEqual({ ok: true });++ expect(recordBackendActivityMock).toHaveBeenCalledOnce();+ expect(recordBackendActivityMock).toHaveBeenCalledWith({+ userId: "user-1",+ organizationId: "org-1",+ });+ });++ it("does not wait for analytics before continuing the API request", async () => {+ recordBackendActivityMock.mockReturnValueOnce(new Promise(() => {}));++ await expect(+ createCaller().project({ projectId: "project-1" }),+ ).resolves.toEqual({ ok: true });+ });++ it("does not record activity for a project outside the user's session", async () => {+ await expect(+ createCaller().project({ projectId: "project-other" }),+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });++ expect(recordBackendActivityMock).not.toHaveBeenCalled();+ });++ it("does not record activity for an unauthenticated request", async () => {+ await expect(+ createCaller(null).project({ projectId: "project-1" }),+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });++ expect(recordBackendActivityMock).not.toHaveBeenCalled();+ });+});diff --git a/web/src/__tests__/server/unit/backendActivity.servertest.ts b/web/src/__tests__/server/unit/backendActivity.servertest.tsnew file mode 100644index 0000000000..66ab9ec267--- /dev/null+++ b/web/src/__tests__/server/unit/backendActivity.servertest.ts@@ -0,0 +1,209 @@+import { describe, expect, it, vi } from "vitest";++vi.mock("@langfuse/shared/src/server", () => ({+ logger: { debug: vi.fn(), warn: vi.fn() },+ redis: null,+ ClickHouseClientManager: {+ getInstance: () => ({ closeAllConnections: vi.fn() }),+ },+}));++vi.mock("@/src/env.mjs", () => ({+ env: { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "US" },+}));++vi.mock("@/src/features/posthog-analytics/ServerPosthog", () => ({+ ServerPosthog: class {+ capture() {}+ },+}));++import { createBackendActivityTracker } from "@/src/features/posthog-analytics/server/backendActivity";++const createTracker = (+ options: {+ cloudRegion?: string;+ now?: Date;+ wasFirstActivity?: boolean;+ } = {},+) => {+ const cloudRegion = Object.hasOwn(options, "cloudRegion")+ ? options.cloudRegion+ : "US";+ const now = options.now ?? new Date("2026-08-10T12:00:00.000Z");+ const wasFirstActivity = options.wasFirstActivity ?? true;+ const capture = vi.fn();+ const setIfAbsent = vi.fn().mockResolvedValue(wasFirstActivity);+ const tracker = createBackendActivityTracker({+ capture,+ cloudRegion,+ now: () => now,+ setIfAbsent,+ });++ return { capture, setIfAbsent, tracker };+};++describe("backend activity tracking", () => {+ it("captures a project-scoped activity event once per UTC day", async () => {+ const { capture, setIfAbsent, tracker } = createTracker();++ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });+ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });++ expect(setIfAbsent).toHaveBeenCalledOnce();+ expect(setIfAbsent).toHaveBeenCalledWith(+ "langfuse:backend-activity:2026-08-10:user-1:project:project-1",+ 172_800,+ );+ expect(capture).toHaveBeenCalledOnce();+ expect(capture).toHaveBeenCalledWith({+ distinctId: "user-1",+ event: "backend:activity",+ properties: {+ activityScope: "project",+ cloudRegion: "US",+ organizationId: "org-1",+ projectId: "project-1",+ },+ timestamp: new Date("2026-08-10T12:00:00.000Z"),+ disableGeoip: true,+ });+ });++ it("tracks organization activity separately from project activity", async () => {+ const { capture, setIfAbsent, tracker } = createTracker();++ await tracker({ userId: "user-1", organizationId: "org-1" });+ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });++ expect(setIfAbsent).toHaveBeenCalledTimes(2);+ expect(capture).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({+ properties: {+ activityScope: "organization",+ cloudRegion: "US",+ organizationId: "org-1",+ },+ }),+ );+ expect(capture).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({+ properties: {+ activityScope: "project",+ cloudRegion: "US",+ organizationId: "org-1",+ projectId: "project-1",+ },+ }),+ );+ });++ it("captures the same project again on the next UTC day", async () => {+ let now = new Date("2026-08-10T23:59:59.000Z");+ const capture = vi.fn();+ const setIfAbsent = vi.fn().mockResolvedValue(true);+ const tracker = createBackendActivityTracker({+ capture,+ cloudRegion: "US",+ now: () => now,+ setIfAbsent,+ });++ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });+ now = new Date("2026-08-11T00:00:01.000Z");+ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });++ expect(setIfAbsent).toHaveBeenCalledTimes(2);+ expect(setIfAbsent).toHaveBeenNthCalledWith(+ 1,+ "langfuse:backend-activity:2026-08-10:user-1:project:project-1",+ 172_800,+ );+ expect(setIfAbsent).toHaveBeenNthCalledWith(+ 2,+ "langfuse:backend-activity:2026-08-11:user-1:project:project-1",+ 172_800,+ );+ expect(capture).toHaveBeenCalledTimes(2);+ });++ it("does not capture when another server already recorded the activity", async () => {+ const { capture, setIfAbsent, tracker } = createTracker({+ wasFirstActivity: false,+ });++ await tracker({ userId: "user-1", organizationId: "org-1" });+ await tracker({ userId: "user-1", organizationId: "org-1" });++ expect(setIfAbsent).toHaveBeenCalledOnce();+ expect(capture).not.toHaveBeenCalled();+ });++ it("does not track outside Langfuse Cloud", async () => {+ const { capture, setIfAbsent, tracker } = createTracker({+ cloudRegion: undefined,+ });++ await tracker({+ userId: "user-1",+ organizationId: "org-1",+ projectId: "project-1",+ });++ expect(setIfAbsent).not.toHaveBeenCalled();+ expect(capture).not.toHaveBeenCalled();+ });++ it("keeps tracking failures off the authenticated API path", async () => {+ const capture = vi.fn();+ const tracker = createBackendActivityTracker({+ capture,+ cloudRegion: "US",+ now: () => new Date("2026-08-10T12:00:00.000Z"),+ setIfAbsent: vi.fn().mockRejectedValue(new Error("Redis unavailable")),+ });++ await expect(+ tracker({ userId: "user-1", organizationId: "org-1" }),+ ).resolves.toBeUndefined();+ expect(capture).not.toHaveBeenCalled();+ });++ it("keeps PostHog capture failures off the authenticated API path", async () => {+ const tracker = createBackendActivityTracker({+ capture: vi.fn(() => {+ throw new Error("PostHog unavailable");+ }),+ cloudRegion: "US",+ now: () => new Date("2026-08-10T12:00:00.000Z"),+ setIfAbsent: vi.fn().mockResolvedValue(true),+ });++ await expect(+ tracker({ userId: "user-1", organizationId: "org-1" }),+ ).resolves.toBeUndefined();+ });+});diff --git a/web/src/features/posthog-analytics/server/backendActivity.ts b/web/src/features/posthog-analytics/server/backendActivity.tsnew file mode 100644index 0000000000..3f67d6ded6--- /dev/null+++ b/web/src/features/posthog-analytics/server/backendActivity.ts@@ -0,0 +1,107 @@+import { logger, redis } from "@langfuse/shared/src/server";++import { env } from "@/src/env.mjs";+import { ServerPosthog } from "@/src/features/posthog-analytics/ServerPosthog";++const BACKEND_ACTIVITY_EVENT = "backend:activity";+const DEDUPLICATION_TTL_SECONDS = 2 * 24 * 60 * 60;+const LOCAL_CACHE_MAX_ENTRIES = 100_000;++type BackendActivity = {+ userId: string;+ organizationId: string;+ projectId?: string;+};++type BackendActivityTrackerDependencies = {+ capture: ServerPosthog["capture"];+ cloudRegion?: string;+ now: () => Date;+ setIfAbsent: (key: string, ttlSeconds: number) => Promise<boolean>;+};++export const createBackendActivityTracker = ({+ capture,+ cloudRegion,+ now,+ setIfAbsent,+}: BackendActivityTrackerDependencies) => {+ const localCache = new Set<string>();++ const cacheLocally = (key: string) => {+ if (localCache.size >= LOCAL_CACHE_MAX_ENTRIES) {+ const oldestKey = localCache.values().next().value;+ if (oldestKey) localCache.delete(oldestKey);+ }+ localCache.add(key);+ };++ return async ({+ userId,+ organizationId,+ projectId,+ }: BackendActivity): Promise<void> => {+ if (!cloudRegion) return;++ const timestamp = now();+ const activityScope = projectId ? "project" : "organization";+ const scopeId = projectId ?? organizationId;+ const utcDate = timestamp.toISOString().slice(0, 10);+ const deduplicationKey = [+ "langfuse",+ "backend-activity",+ utcDate,+ userId,+ activityScope,+ scopeId,+ ].join(":");++ if (localCache.has(deduplicationKey)) return;+ // Mark the attempt before the first await so concurrent requests and Redis+ // outages cannot create an analytics retry storm in this web process.+ cacheLocally(deduplicationKey);++ try {+ const isFirstActivity = await setIfAbsent(+ deduplicationKey,+ DEDUPLICATION_TTL_SECONDS,+ );++ if (!isFirstActivity) return;++ capture({+ distinctId: userId,+ event: BACKEND_ACTIVITY_EVENT,+ properties: {+ activityScope,+ cloudRegion,+ organizationId,+ ...(projectId ? { projectId } : {}),+ },+ timestamp,+ disableGeoip: true,+ });+ } catch (error) {+ // Product analytics must never make an authenticated API request fail.+ logger.warn("Failed to record backend activity", { error });+ }+ };+};++let serverPosthog: ServerPosthog | undefined;++export const recordBackendActivity = createBackendActivityTracker({+ capture: (event) => {+ serverPosthog ??= new ServerPosthog();+ serverPosthog.capture(event);+ },+ cloudRegion:+ env.NODE_ENV === "production"+ ? env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION+ : undefined,+ now: () => new Date(),+ setIfAbsent: async (key, ttlSeconds) => {+ if (!redis) return false;+ return (await redis.set(key, "1", "EX", ttlSeconds, "NX")) === "OK";+ },+});diff --git a/web/src/server/api/trpc.ts b/web/src/server/api/trpc.tsindex 6d9d49dfb6..030e93d877 100644--- a/web/src/server/api/trpc.ts+++ b/web/src/server/api/trpc.ts@@ -95,6 +95,7 @@ import { AdminApiAuthService } from "@/src/ee/features/admin-api/server/adminApiimport { env } from "@/src/env.mjs";import { isBaseError, parseIO } from "@langfuse/shared";import { type Flag } from "@/src/features/feature-flags/types";+import { recordBackendActivity } from "@/src/features/posthog-analytics/server/backendActivity";setUpSuperjson();@@ -281,6 +282,14 @@ const inputProjectSchema = z.object({projectId: z.string(),});+const trackPosthogActivity = (+ activity: Parameters<typeof recordBackendActivity>[0],+) => {+ recordBackendActivity(activity).catch((error) => {+ logger.warn("Failed to track PostHog activity", { error });+ });+};+/*** Protected (authenticated) procedure with project role*/@@ -331,6 +340,11 @@ const enforceUserIsAuthedAndProjectMember = t.middleware(async (opts) => {projectId,orgId: dbProject.orgId,});+ trackPosthogActivity({+ userId: ctx.session.user.id,+ organizationId: dbProject.orgId,+ projectId,+ });return next({ctx: {// infers the `session` as non-nullable@@ -361,6 +375,12 @@ const enforceUserIsAuthedAndProjectMember = t.middleware(async (opts) => {});}+ trackPosthogActivity({+ userId: ctx.session.user.id,+ organizationId: sessionProject.organization.id,+ projectId,+ });+return next({ctx: {// infers the `session` as non-nullable@@ -456,6 +476,11 @@ const enforceIsAuthedAndOrgMember = t.middleware(async (opts) => {});}+ trackPosthogActivity({+ userId: ctx.session.user.id,+ organizationId: orgId,+ });+return next({ctx: {session: {