🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(posthog): disable product analytics in the HIPAA region (#16721)
Max Deichmann95933f73a668
diff --git a/.agents/skills/posthog-instrumentation/SKILL.md b/.agents/skills/posthog-instrumentation/SKILL.mdindex c677510f36..0157902f95 100644--- a/.agents/skills/posthog-instrumentation/SKILL.md+++ b/.agents/skills/posthog-instrumentation/SKILL.md@@ -57,6 +57,16 @@ should emit an event — do not skip the question silently.`ServerPosthog` (e.g. `cloud_signup_complete`, playground analytics) withevent names distinct from any frontend event — keep it that way; reusing aname across client and server double-counts.+- **The HIPAA region runs NO product analytics.** Both entry points gate on+ [`productAnalyticsAvailability.ts`](../../../web/src/features/posthog-analytics/productAnalyticsAvailability.ts):+ the browser SDK (`posthog-js`) is only initialized when+ `getPostHogClientConfig()` returns a config — `init()` itself fetches remote+ config, so skip it rather than opting out after. The server SDK+ (`posthog-node`, via `ServerPosthog`) is constructed as usual, then+ `disable()`d when `isProductAnalyticsAvailable()` is false. Missing env vars+ are NOT a gate — `ServerPosthog` falls back to Langfuse's telemetry key.+ Never add a per-call-site region list, and never construct a PostHog client+ that bypasses that module.## The 6 rules (each earned the hard way on LFE-10781 / PR #14929)@@ -127,8 +137,9 @@ renderer that can display customer-controlled data:1. **Allowlist eligible deployments.** Keep replay disabled by default andenable it only in explicitly approved hosted regions. A configured PostHogkey is not proof that a deployment is eligible. In Langfuse, replay remains- disabled when the cloud region is absent and in the HIPAA region; cover an- eligible region, HIPAA, and no cloud region in config tests.+ disabled when the cloud region is absent, and the HIPAA region initializes no+ PostHog client at all; cover an eligible region, HIPAA, and no cloud region+ in config tests.2. **Classify by data provenance, not HTML element.** Customer-controlled dataincludes trace/observation payloads, dataset items, prompts, generatedoutput, evaluator/code input, identifiers, comments, names, tags, metadata,diff --git a/web/src/__tests__/posthog-product-analytics-region.clienttest.ts b/web/src/__tests__/posthog-product-analytics-region.clienttest.tsnew file mode 100644index 0000000000..fd8d12eb0b--- /dev/null+++ b/web/src/__tests__/posthog-product-analytics-region.clienttest.ts@@ -0,0 +1,75 @@+// @vitest-environment jsdom++/**+ * Product-analytics region gate (browser half).+ *+ * The HIPAA cloud region runs no Langfuse product analytics at all, so the+ * PostHog browser SDK must never be initialized there — not even when the+ * deployment carries a PostHog key/host. Every other deployment keeps its+ * current behaviour, which is what the positive cases below pin down.+ */++import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const { postHogInitMock } = vi.hoisted(() => ({+ postHogInitMock: vi.fn(),+}));++vi.mock("posthog-js", () => ({+ default: {+ init: postHogInitMock,+ },+}));++vi.mock("posthog-js/react", () => ({+ PostHogProvider: vi.fn(),+}));++const importAppShell = async () => {+ vi.resetModules();+ await import("@/src/pages/_app");+};++// Every region that keeps product analytics, plus self-hosted (region unset).+const ANALYTICS_ENABLED_REGIONS: (string | undefined)[] = [+ "US",+ "EU",+ "JP",+ "STAGING",+ undefined,+];++describe("PostHog product analytics region gate", () => {+ beforeEach(() => {+ postHogInitMock.mockClear();+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://eu.i.posthog.com");+ });++ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it("never initializes PostHog in the HIPAA cloud region", async () => {+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");++ await importAppShell();++ expect(postHogInitMock).not.toHaveBeenCalled();+ });++ it.each(ANALYTICS_ENABLED_REGIONS)(+ "initializes PostHog in region %s",+ async (region) => {+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", region);++ await importAppShell();++ expect(postHogInitMock).toHaveBeenCalledTimes(1);+ expect(postHogInitMock).toHaveBeenCalledWith(+ "phc_test",+ expect.objectContaining({ api_host: "https://eu.i.posthog.com" }),+ );+ },+ );+});diff --git a/web/src/__tests__/posthog-session-replay-mask.clienttest.ts b/web/src/__tests__/posthog-session-replay-mask.clienttest.tsindex c7331fcf53..74e0aa87f9 100644--- a/web/src/__tests__/posthog-session-replay-mask.clienttest.ts+++ b/web/src/__tests__/posthog-session-replay-mask.clienttest.ts@@ -42,16 +42,17 @@ describe("PostHog session replay privacy", () => {expect(config.disable_session_recording).toBe(false);});- it("disables session recording in the HIPAA cloud region", async () => {+ // Stronger than disabling the recorder: the HIPAA region runs no product+ // analytics at all, so there is no PostHog client to record with. The wider+ // region gate lives in posthog-product-analytics-region.clienttest.ts.+ it("initializes no PostHog client in the HIPAA cloud region", async () => {vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://us.i.posthog.com");vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");await import("@/src/pages/_app");- expect(initMock).toHaveBeenCalledTimes(1);- const config = initMock.mock.calls[0]![1] as Partial<PostHogConfig>;- expect(config.disable_session_recording).toBe(true);+ expect(initMock).not.toHaveBeenCalled();});it("disables session recording outside Langfuse Cloud", async () => {diff --git a/web/src/__tests__/server/unit/serverPosthog.servertest.ts b/web/src/__tests__/server/unit/serverPosthog.servertest.tsnew file mode 100644index 0000000000..d6547a5b95--- /dev/null+++ b/web/src/__tests__/server/unit/serverPosthog.servertest.ts@@ -0,0 +1,104 @@+import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";++/**+ * Product-analytics region gate (server half).+ *+ * `ServerPosthog` is the single server-side entry point for Langfuse product+ * analytics (signup conversion, backend activity, self-host telemetry). In the+ * HIPAA cloud region it still constructs a `posthog-node` client, then opts it+ * out with `disable()` so capture becomes a no-op. That is enough on the+ * server: unlike the browser SDK, disable does not phone home.+ */++const {+ postHogConstructor,+ captureMock,+ disableMock,+ shutdownMock,+ flushMock,+} = vi.hoisted(() => ({+ postHogConstructor: vi.fn(),+ captureMock: vi.fn(),+ disableMock: vi.fn().mockResolvedValue(undefined),+ shutdownMock: vi.fn().mockResolvedValue(undefined),+ flushMock: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("posthog-node", () => ({+ PostHog: class {+ constructor(...args: unknown[]) {+ postHogConstructor(...args);+ }+ capture = (...args: unknown[]) => {+ if (disableMock.mock.calls.length > 0) return;+ captureMock(...args);+ };+ disable = disableMock;+ shutdown = shutdownMock;+ flush = flushMock;+ debug = vi.fn();+ },+}));++const loadServerPosthog = async () => {+ vi.resetModules();+ const { ServerPosthog } =+ await import("@/src/features/posthog-analytics/ServerPosthog");+ return new ServerPosthog();+};++const captureArgs = {+ distinctId: "user-1",+ event: "backend:activity",+} as const;++describe("ServerPosthog product analytics region gate", () => {+ beforeEach(() => {+ postHogConstructor.mockClear();+ captureMock.mockClear();+ disableMock.mockClear();+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://eu.i.posthog.com");+ });++ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it("disables the client in the HIPAA cloud region so capture is a no-op", async () => {+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");++ const posthog = await loadServerPosthog();+ posthog.capture(captureArgs);+ await posthog.flush();+ await posthog.shutdown();++ expect(postHogConstructor).toHaveBeenCalledTimes(1);+ expect(disableMock).toHaveBeenCalledTimes(1);+ expect(captureMock).not.toHaveBeenCalled();+ });++ it("still disables the HIPAA client when it is built from the telemetry fallback", async () => {+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", undefined);+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", undefined);++ const posthog = await loadServerPosthog();+ posthog.capture(captureArgs);++ expect(postHogConstructor).toHaveBeenCalledTimes(1);+ expect(disableMock).toHaveBeenCalledTimes(1);+ expect(captureMock).not.toHaveBeenCalled();+ });++ it.each([["US"], ["EU"], ["JP"]])("keeps capturing in %s", async (region) => {+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", region);++ const posthog = await loadServerPosthog();+ posthog.capture(captureArgs);++ expect(postHogConstructor).toHaveBeenCalledTimes(1);+ expect(disableMock).not.toHaveBeenCalled();+ expect(captureMock).toHaveBeenCalledWith(captureArgs);+ });+});diff --git a/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts b/web/src/features/auth/lib/createProjectMembershipsOnSignup.tsindex 71b27ac745..1ca08bd673 100644--- a/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts+++ b/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts@@ -284,15 +284,13 @@ export async function createProjectMembershipsOnSignup(// for conversion metric tracking in posthog: did a new user sign up?// Fires on all production cloud regions, including ones added in the// future. STAGING/DEV are excluded to keep test signups out of- // conversion metrics, HIPAA is excluded deliberately to keep product- // analytics off that deployment, and self-hosted deployments never emit- // this event as the region env is unset.+ // conversion metrics, and self-hosted deployments never emit this+ // event as the region env is unset. HIPAA still constructs ServerPosthog+ // here; the client is opted out via disable() so the capture is a no-op.if (isNewUser &&env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&- !["STAGING", "DEV", "HIPAA"].includes(- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,- )+ !["STAGING", "DEV"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)) {try {const posthog = new ServerPosthog();diff --git a/web/src/features/auth/lib/signOut.ts b/web/src/features/auth/lib/signOut.tsindex f310e8dd4d..a32634fac6 100644--- a/web/src/features/auth/lib/signOut.ts+++ b/web/src/features/auth/lib/signOut.ts@@ -1,6 +1,7 @@import { signOut } from "next-auth/react";import posthog from "posthog-js";import { env } from "@/src/env.mjs";+import { isPostHogClientEnabled } from "@/src/features/posthog-analytics/productAnalyticsAvailability";import { clearV4BetaEnabledSentryTag } from "@/src/utils/sentryV4BetaTag";/**@@ -19,7 +20,7 @@ export const signOutCleanly = async () => {// see `unauthenticated`. Drop the pageload cache here so the sign-in// hard load is not tagged with the previous user's v4 state.clearV4BetaEnabledSentryTag();- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {+ if (isPostHogClientEnabled()) {posthog.reset();}// On preview deployments the sign-in page signs visitors back indiff --git a/web/src/features/posthog-analytics/ServerPosthog.ts b/web/src/features/posthog-analytics/ServerPosthog.tsindex 9162879417..10ea77f891 100644--- a/web/src/features/posthog-analytics/ServerPosthog.ts+++ b/web/src/features/posthog-analytics/ServerPosthog.ts@@ -1,4 +1,5 @@import { env } from "@/src/env.mjs";+import { isProductAnalyticsAvailable } from "@/src/features/posthog-analytics/productAnalyticsAvailability";import { PostHog } from "posthog-node";const FALLBACK_POSTHOG_KEY = "phc_zkMwFajk8ehObUlMth0D7DtPItFnxETi3lmSvyQDrwB";@@ -6,6 +7,7 @@ const FALLBACK_POSTHOG_HOST = "https://eu.posthog.com";export class ServerPosthog {private posthog: PostHog | null;+ private optOut: Promise<void> | undefined;constructor() {const telemetryEnabled = env.TELEMETRY_ENABLED !== "false";@@ -20,6 +22,13 @@ export class ServerPosthog {if (apiKey && host) {this.posthog = new PostHog(apiKey, { host });if (process.env.NODE_ENV === "development") this.posthog.debug();+ // Unlike the browser SDK, posthog-node disable() is a local opt-out:+ // capture becomes a no-op and nothing is sent. HIPAA uses this instead+ // of skipping construction. The flag flips synchronously; the promise is+ // kept so the constructor does not float it.+ if (!isProductAnalyticsAvailable()) {+ this.optOut = this.posthog.disable();+ }} else {this.posthog = null;}@@ -30,6 +39,7 @@ export class ServerPosthog {}async shutdown() {+ await this.optOut;await this.posthog?.shutdown();}diff --git a/web/src/features/posthog-analytics/productAnalyticsAvailability.ts b/web/src/features/posthog-analytics/productAnalyticsAvailability.tsnew file mode 100644index 0000000000..daa1a40697--- /dev/null+++ b/web/src/features/posthog-analytics/productAnalyticsAvailability.ts@@ -0,0 +1,23 @@+// PostHog product analytics are off in the HIPAA cloud region. Not setting the+// env vars is not enough: ServerPosthog falls back to Langfuse's telemetry key.+//+// Read from process.env, not the validated env object, so this also works at+// module scope in the app shell where posthog-js is initialized. Next.js inlines+// NEXT_PUBLIC_* per region build (see web/Dockerfile).++export const isProductAnalyticsAvailable = (): boolean =>+ process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== "HIPAA";++/** Key/host for posthog-js, or null when the browser SDK must not initialize. */+export const getPostHogClientConfig = (): {+ key: string;+ host: string;+} | null => {+ const key = process.env.NEXT_PUBLIC_POSTHOG_KEY;+ const host = process.env.NEXT_PUBLIC_POSTHOG_HOST;+ if (!isProductAnalyticsAvailable() || !key || !host) return null;+ return { key, host };+};++export const isPostHogClientEnabled = (): boolean =>+ getPostHogClientConfig() !== null;diff --git a/web/src/pages/_app.tsx b/web/src/pages/_app.tsxindex 51cf036c78..f67c69a0f1 100644--- a/web/src/pages/_app.tsx+++ b/web/src/pages/_app.tsx@@ -92,26 +92,31 @@ import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";import { ScoreCacheProvider } from "@/src/features/scores/contexts/ScoreCacheContext";import { CorrectionCacheProvider } from "@/src/features/corrections/contexts/CorrectionCacheContext";import { V4_BETA_ENABLED_POSTHOG_PROPERTY } from "@/src/features/posthog-analytics/usePostHogClientCapture";+import {+ getPostHogClientConfig,+ isPostHogClientEnabled,+ isProductAnalyticsAvailable,+} from "@/src/features/posthog-analytics/productAnalyticsAvailability";+// Session replay is a Langfuse Cloud feature, so self-hosted never records.+// The product-analytics gate makes this redundant in HIPAA (PostHog is not+// initialized there at all); it stays in the expression as defense in depth for+// a compliance-sensitive flag.const isPostHogSessionRecordingEnabled =env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined &&- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== "HIPAA";+ isProductAnalyticsAvailable();-// Check that PostHog is client-side (used to handle Next.js SSR) and that env vars are set-if (- typeof window !== "undefined" &&- process.env.NEXT_PUBLIC_POSTHOG_KEY &&- process.env.NEXT_PUBLIC_POSTHOG_HOST-) {- posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY, {- api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://eu.posthog.com",+// Check that PostHog is client-side (used to handle Next.js SSR), that env vars+// are set, and that the deployment region runs product analytics at all.+const postHogClientConfig = getPostHogClientConfig();+if (typeof window !== "undefined" && postHogClientConfig) {+ posthog.init(postHogClientConfig.key, {+ api_host: postHogClientConfig.host,ui_host: "https://eu.posthog.com",// Enable debug mode in developmentloaded: (posthog) => {if (process.env.NODE_ENV === "development") posthog.debug();},- // Session replay is a Langfuse Cloud feature and must stay disabled in the- // HIPAA region. Other PostHog analytics remain independently configured.disable_session_recording: !isPostHogSessionRecordingEnabled,session_recording: {maskAllInputs: true,@@ -144,7 +149,7 @@ const MyApp: AppType<{ session: Session | null }> = ({useEffect(() => {// PostHog (cloud.langfuse.com)- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {+ if (isPostHogClientEnabled()) {const handleRouteChange = () => {posthog.capture("$pageview");};@@ -247,7 +252,7 @@ function UserTracking() {) {lastIdentifiedUser.current = JSON.stringify(sessionUser);// PostHog- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {+ if (isPostHogClientEnabled()) {posthog.identify(sessionUser.id ?? undefined, {environment: process.env.NODE_ENV,email: sessionUser.email ?? undefined,