Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(analytics): track authenticated backend activity (#15963)

Max Deichmanncc4904f685f7
diff --git a/web/src/__tests__/server/backend-activity-middleware.servertest.ts b/web/src/__tests__/server/backend-activity-middleware.servertest.ts
new file mode 100644
index 0000000000..6eb03f240c
--- /dev/null
+++ b/web/src/__tests__/server/backend-activity-middleware.servertest.ts
@@ -0,0 +1,116 @@
+import type { Session } from "next-auth";
+import * as z from "zod";
+
+const { recordBackendActivityMock } = vi.hoisted(() => ({
+ recordBackendActivityMock: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@/src/features/posthog-analytics/server/backendActivity", () => ({
+ recordBackendActivity: recordBackendActivityMock,
+}));
+
+import {
+ createInnerTRPCContext,
+ createTRPCRouter,
+ protectedOrganizationProcedure,
+ protectedProjectProcedureWithoutTracing,
+} from "@/src/server/api/trpc";
+
+const activityRouter = createTRPCRouter({
+ project: protectedProjectProcedureWithoutTracing
+ .input(z.object({ projectId: z.string() }))
+ .query(() => ({ ok: true })),
+ organization: protectedOrganizationProcedure
+ .input(z.object({ orgId: z.string() }))
+ .query(() => ({ ok: true })),
+});
+
+const session = {
+ expires: "1",
+ user: {
+ id: "user-1",
+ email: "[email protected]",
+ name: "Example User",
+ admin: false,
+ canCreateOrganizations: true,
+ featureFlags: {},
+ organizations: [
+ {
+ id: "org-1",
+ name: "Example Organization",
+ role: "MEMBER",
+ plan: "cloud:hobby",
+ cloudConfig: undefined,
+ metadata: {},
+ projects: [
+ {
+ id: "project-1",
+ name: "Example Project",
+ role: "MEMBER",
+ retentionDays: 30,
+ deletedAt: null,
+ },
+ ],
+ },
+ ],
+ },
+ environment: {},
+} as Session;
+
+const createCaller = (callerSession: Session | null = session) =>
+ activityRouter.createCaller(
+ createInnerTRPCContext({ session: callerSession, headers: {} }),
+ );
+
+describe("backend activity authorization middleware", () => {
+ beforeEach(() => vi.clearAllMocks());
+
+ it("records server-derived user, organization, and project ids after project authorization", async () => {
+ await expect(
+ createCaller().project({ projectId: "project-1" }),
+ ).resolves.toEqual({ ok: true });
+
+ expect(recordBackendActivityMock).toHaveBeenCalledOnce();
+ expect(recordBackendActivityMock).toHaveBeenCalledWith({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+ });
+
+ it("records organization activity after organization authorization", async () => {
+ await expect(
+ createCaller().organization({ orgId: "org-1" }),
+ ).resolves.toEqual({ ok: true });
+
+ expect(recordBackendActivityMock).toHaveBeenCalledOnce();
+ expect(recordBackendActivityMock).toHaveBeenCalledWith({
+ userId: "user-1",
+ organizationId: "org-1",
+ });
+ });
+
+ it("does not wait for analytics before continuing the API request", async () => {
+ recordBackendActivityMock.mockReturnValueOnce(new Promise(() => {}));
+
+ await expect(
+ createCaller().project({ projectId: "project-1" }),
+ ).resolves.toEqual({ ok: true });
+ });
+
+ it("does not record activity for a project outside the user's session", async () => {
+ await expect(
+ createCaller().project({ projectId: "project-other" }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+
+ expect(recordBackendActivityMock).not.toHaveBeenCalled();
+ });
+
+ it("does not record activity for an unauthenticated request", async () => {
+ await expect(
+ createCaller(null).project({ projectId: "project-1" }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+
+ expect(recordBackendActivityMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/src/__tests__/server/unit/backendActivity.servertest.ts b/web/src/__tests__/server/unit/backendActivity.servertest.ts
new file mode 100644
index 0000000000..66ab9ec267
--- /dev/null
+++ b/web/src/__tests__/server/unit/backendActivity.servertest.ts
@@ -0,0 +1,209 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { debug: vi.fn(), warn: vi.fn() },
+ redis: null,
+ ClickHouseClientManager: {
+ getInstance: () => ({ closeAllConnections: vi.fn() }),
+ },
+}));
+
+vi.mock("@/src/env.mjs", () => ({
+ env: { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "US" },
+}));
+
+vi.mock("@/src/features/posthog-analytics/ServerPosthog", () => ({
+ ServerPosthog: class {
+ capture() {}
+ },
+}));
+
+import { createBackendActivityTracker } from "@/src/features/posthog-analytics/server/backendActivity";
+
+const createTracker = (
+ options: {
+ cloudRegion?: string;
+ now?: Date;
+ wasFirstActivity?: boolean;
+ } = {},
+) => {
+ const cloudRegion = Object.hasOwn(options, "cloudRegion")
+ ? options.cloudRegion
+ : "US";
+ const now = options.now ?? new Date("2026-08-10T12:00:00.000Z");
+ const wasFirstActivity = options.wasFirstActivity ?? true;
+ const capture = vi.fn();
+ const setIfAbsent = vi.fn().mockResolvedValue(wasFirstActivity);
+ const tracker = createBackendActivityTracker({
+ capture,
+ cloudRegion,
+ now: () => now,
+ setIfAbsent,
+ });
+
+ return { capture, setIfAbsent, tracker };
+};
+
+describe("backend activity tracking", () => {
+ it("captures a project-scoped activity event once per UTC day", async () => {
+ const { capture, setIfAbsent, tracker } = createTracker();
+
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+
+ expect(setIfAbsent).toHaveBeenCalledOnce();
+ expect(setIfAbsent).toHaveBeenCalledWith(
+ "langfuse:backend-activity:2026-08-10:user-1:project:project-1",
+ 172_800,
+ );
+ expect(capture).toHaveBeenCalledOnce();
+ expect(capture).toHaveBeenCalledWith({
+ distinctId: "user-1",
+ event: "backend:activity",
+ properties: {
+ activityScope: "project",
+ cloudRegion: "US",
+ organizationId: "org-1",
+ projectId: "project-1",
+ },
+ timestamp: new Date("2026-08-10T12:00:00.000Z"),
+ disableGeoip: true,
+ });
+ });
+
+ it("tracks organization activity separately from project activity", async () => {
+ const { capture, setIfAbsent, tracker } = createTracker();
+
+ await tracker({ userId: "user-1", organizationId: "org-1" });
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+
+ expect(setIfAbsent).toHaveBeenCalledTimes(2);
+ expect(capture).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({
+ properties: {
+ activityScope: "organization",
+ cloudRegion: "US",
+ organizationId: "org-1",
+ },
+ }),
+ );
+ expect(capture).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({
+ properties: {
+ activityScope: "project",
+ cloudRegion: "US",
+ organizationId: "org-1",
+ projectId: "project-1",
+ },
+ }),
+ );
+ });
+
+ it("captures the same project again on the next UTC day", async () => {
+ let now = new Date("2026-08-10T23:59:59.000Z");
+ const capture = vi.fn();
+ const setIfAbsent = vi.fn().mockResolvedValue(true);
+ const tracker = createBackendActivityTracker({
+ capture,
+ cloudRegion: "US",
+ now: () => now,
+ setIfAbsent,
+ });
+
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+ now = new Date("2026-08-11T00:00:01.000Z");
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+
+ expect(setIfAbsent).toHaveBeenCalledTimes(2);
+ expect(setIfAbsent).toHaveBeenNthCalledWith(
+ 1,
+ "langfuse:backend-activity:2026-08-10:user-1:project:project-1",
+ 172_800,
+ );
+ expect(setIfAbsent).toHaveBeenNthCalledWith(
+ 2,
+ "langfuse:backend-activity:2026-08-11:user-1:project:project-1",
+ 172_800,
+ );
+ expect(capture).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not capture when another server already recorded the activity", async () => {
+ const { capture, setIfAbsent, tracker } = createTracker({
+ wasFirstActivity: false,
+ });
+
+ await tracker({ userId: "user-1", organizationId: "org-1" });
+ await tracker({ userId: "user-1", organizationId: "org-1" });
+
+ expect(setIfAbsent).toHaveBeenCalledOnce();
+ expect(capture).not.toHaveBeenCalled();
+ });
+
+ it("does not track outside Langfuse Cloud", async () => {
+ const { capture, setIfAbsent, tracker } = createTracker({
+ cloudRegion: undefined,
+ });
+
+ await tracker({
+ userId: "user-1",
+ organizationId: "org-1",
+ projectId: "project-1",
+ });
+
+ expect(setIfAbsent).not.toHaveBeenCalled();
+ expect(capture).not.toHaveBeenCalled();
+ });
+
+ it("keeps tracking failures off the authenticated API path", async () => {
+ const capture = vi.fn();
+ const tracker = createBackendActivityTracker({
+ capture,
+ cloudRegion: "US",
+ now: () => new Date("2026-08-10T12:00:00.000Z"),
+ setIfAbsent: vi.fn().mockRejectedValue(new Error("Redis unavailable")),
+ });
+
+ await expect(
+ tracker({ userId: "user-1", organizationId: "org-1" }),
+ ).resolves.toBeUndefined();
+ expect(capture).not.toHaveBeenCalled();
+ });
+
+ it("keeps PostHog capture failures off the authenticated API path", async () => {
+ const tracker = createBackendActivityTracker({
+ capture: vi.fn(() => {
+ throw new Error("PostHog unavailable");
+ }),
+ cloudRegion: "US",
+ now: () => new Date("2026-08-10T12:00:00.000Z"),
+ setIfAbsent: vi.fn().mockResolvedValue(true),
+ });
+
+ await expect(
+ tracker({ userId: "user-1", organizationId: "org-1" }),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/web/src/features/posthog-analytics/server/backendActivity.ts b/web/src/features/posthog-analytics/server/backendActivity.ts
new file mode 100644
index 0000000000..3f67d6ded6
--- /dev/null
+++ b/web/src/features/posthog-analytics/server/backendActivity.ts
@@ -0,0 +1,107 @@
+import { logger, redis } from "@langfuse/shared/src/server";
+
+import { env } from "@/src/env.mjs";
+import { ServerPosthog } from "@/src/features/posthog-analytics/ServerPosthog";
+
+const BACKEND_ACTIVITY_EVENT = "backend:activity";
+const DEDUPLICATION_TTL_SECONDS = 2 * 24 * 60 * 60;
+const LOCAL_CACHE_MAX_ENTRIES = 100_000;
+
+type BackendActivity = {
+ userId: string;
+ organizationId: string;
+ projectId?: string;
+};
+
+type BackendActivityTrackerDependencies = {
+ capture: ServerPosthog["capture"];
+ cloudRegion?: string;
+ now: () => Date;
+ setIfAbsent: (key: string, ttlSeconds: number) => Promise<boolean>;
+};
+
+export const createBackendActivityTracker = ({
+ capture,
+ cloudRegion,
+ now,
+ setIfAbsent,
+}: BackendActivityTrackerDependencies) => {
+ const localCache = new Set<string>();
+
+ const cacheLocally = (key: string) => {
+ if (localCache.size >= LOCAL_CACHE_MAX_ENTRIES) {
+ const oldestKey = localCache.values().next().value;
+ if (oldestKey) localCache.delete(oldestKey);
+ }
+ localCache.add(key);
+ };
+
+ return async ({
+ userId,
+ organizationId,
+ projectId,
+ }: BackendActivity): Promise<void> => {
+ if (!cloudRegion) return;
+
+ const timestamp = now();
+ const activityScope = projectId ? "project" : "organization";
+ const scopeId = projectId ?? organizationId;
+ const utcDate = timestamp.toISOString().slice(0, 10);
+ const deduplicationKey = [
+ "langfuse",
+ "backend-activity",
+ utcDate,
+ userId,
+ activityScope,
+ scopeId,
+ ].join(":");
+
+ if (localCache.has(deduplicationKey)) return;
+ // Mark the attempt before the first await so concurrent requests and Redis
+ // outages cannot create an analytics retry storm in this web process.
+ cacheLocally(deduplicationKey);
+
+ try {
+ const isFirstActivity = await setIfAbsent(
+ deduplicationKey,
+ DEDUPLICATION_TTL_SECONDS,
+ );
+
+ if (!isFirstActivity) return;
+
+ capture({
+ distinctId: userId,
+ event: BACKEND_ACTIVITY_EVENT,
+ properties: {
+ activityScope,
+ cloudRegion,
+ organizationId,
+ ...(projectId ? { projectId } : {}),
+ },
+ timestamp,
+ disableGeoip: true,
+ });
+ } catch (error) {
+ // Product analytics must never make an authenticated API request fail.
+ logger.warn("Failed to record backend activity", { error });
+ }
+ };
+};
+
+let serverPosthog: ServerPosthog | undefined;
+
+export const recordBackendActivity = createBackendActivityTracker({
+ capture: (event) => {
+ serverPosthog ??= new ServerPosthog();
+ serverPosthog.capture(event);
+ },
+ cloudRegion:
+ env.NODE_ENV === "production"
+ ? env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ : undefined,
+ now: () => new Date(),
+ setIfAbsent: async (key, ttlSeconds) => {
+ if (!redis) return false;
+ return (await redis.set(key, "1", "EX", ttlSeconds, "NX")) === "OK";
+ },
+});
diff --git a/web/src/server/api/trpc.ts b/web/src/server/api/trpc.ts
index 6d9d49dfb6..030e93d877 100644
--- a/web/src/server/api/trpc.ts
+++ b/web/src/server/api/trpc.ts
@@ -95,6 +95,7 @@ import { AdminApiAuthService } from "@/src/ee/features/admin-api/server/adminApi
import { env } from "@/src/env.mjs";
import { isBaseError, parseIO } from "@langfuse/shared";
import { type Flag } from "@/src/features/feature-flags/types";
+import { recordBackendActivity } from "@/src/features/posthog-analytics/server/backendActivity";
setUpSuperjson();
@@ -281,6 +282,14 @@ const inputProjectSchema = z.object({
projectId: z.string(),
});
+const trackPosthogActivity = (
+ activity: Parameters<typeof recordBackendActivity>[0],
+) => {
+ recordBackendActivity(activity).catch((error) => {
+ logger.warn("Failed to track PostHog activity", { error });
+ });
+};
+
/**
* Protected (authenticated) procedure with project role
*/
@@ -331,6 +340,11 @@ const enforceUserIsAuthedAndProjectMember = t.middleware(async (opts) => {
projectId,
orgId: dbProject.orgId,
});
+ trackPosthogActivity({
+ userId: ctx.session.user.id,
+ organizationId: dbProject.orgId,
+ projectId,
+ });
return next({
ctx: {
// infers the `session` as non-nullable
@@ -361,6 +375,12 @@ const enforceUserIsAuthedAndProjectMember = t.middleware(async (opts) => {
});
}
+ trackPosthogActivity({
+ userId: ctx.session.user.id,
+ organizationId: sessionProject.organization.id,
+ projectId,
+ });
+
return next({
ctx: {
// infers the `session` as non-nullable
@@ -456,6 +476,11 @@ const enforceIsAuthedAndOrgMember = t.middleware(async (opts) => {
});
}
+ trackPosthogActivity({
+ userId: ctx.session.user.id,
+ organizationId: orgId,
+ });
+
return next({
ctx: {
session: {