Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(posthog): disable product analytics in the HIPAA region (#16721)

Max Deichmann95933f73a668
diff --git a/.agents/skills/posthog-instrumentation/SKILL.md b/.agents/skills/posthog-instrumentation/SKILL.md
index c677510f36..0157902f95 100644
--- a/.agents/skills/posthog-instrumentation/SKILL.md
+++ b/.agents/skills/posthog-instrumentation/SKILL.md
@@ -57,6 +57,16 @@ should emit an event — do not skip the question silently.
`ServerPosthog` (e.g. `cloud_signup_complete`, playground analytics) with
event names distinct from any frontend event — keep it that way; reusing a
name across client and server double-counts.
+- **The HIPAA region runs NO product analytics.** Both entry points gate on
+ [`productAnalyticsAvailability.ts`](../../../web/src/features/posthog-analytics/productAnalyticsAvailability.ts):
+ the browser SDK (`posthog-js`) is only initialized when
+ `getPostHogClientConfig()` returns a config — `init()` itself fetches remote
+ config, so skip it rather than opting out after. The server SDK
+ (`posthog-node`, via `ServerPosthog`) is constructed as usual, then
+ `disable()`d when `isProductAnalyticsAvailable()` is false. Missing env vars
+ are NOT a gate — `ServerPosthog` falls back to Langfuse's telemetry key.
+ Never add a per-call-site region list, and never construct a PostHog client
+ that bypasses that module.
## The 6 rules (each earned the hard way on LFE-10781 / PR #14929)
@@ -127,8 +137,9 @@ renderer that can display customer-controlled data:
1. **Allowlist eligible deployments.** Keep replay disabled by default and
enable it only in explicitly approved hosted regions. A configured PostHog
key is not proof that a deployment is eligible. In Langfuse, replay remains
- disabled when the cloud region is absent and in the HIPAA region; cover an
- eligible region, HIPAA, and no cloud region in config tests.
+ disabled when the cloud region is absent, and the HIPAA region initializes no
+ PostHog client at all; cover an eligible region, HIPAA, and no cloud region
+ in config tests.
2. **Classify by data provenance, not HTML element.** Customer-controlled data
includes trace/observation payloads, dataset items, prompts, generated
output, evaluator/code input, identifiers, comments, names, tags, metadata,
diff --git a/web/src/__tests__/posthog-product-analytics-region.clienttest.ts b/web/src/__tests__/posthog-product-analytics-region.clienttest.ts
new file mode 100644
index 0000000000..fd8d12eb0b
--- /dev/null
+++ b/web/src/__tests__/posthog-product-analytics-region.clienttest.ts
@@ -0,0 +1,75 @@
+// @vitest-environment jsdom
+
+/**
+ * Product-analytics region gate (browser half).
+ *
+ * The HIPAA cloud region runs no Langfuse product analytics at all, so the
+ * PostHog browser SDK must never be initialized there — not even when the
+ * deployment carries a PostHog key/host. Every other deployment keeps its
+ * current behaviour, which is what the positive cases below pin down.
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { postHogInitMock } = vi.hoisted(() => ({
+ postHogInitMock: vi.fn(),
+}));
+
+vi.mock("posthog-js", () => ({
+ default: {
+ init: postHogInitMock,
+ },
+}));
+
+vi.mock("posthog-js/react", () => ({
+ PostHogProvider: vi.fn(),
+}));
+
+const importAppShell = async () => {
+ vi.resetModules();
+ await import("@/src/pages/_app");
+};
+
+// Every region that keeps product analytics, plus self-hosted (region unset).
+const ANALYTICS_ENABLED_REGIONS: (string | undefined)[] = [
+ "US",
+ "EU",
+ "JP",
+ "STAGING",
+ undefined,
+];
+
+describe("PostHog product analytics region gate", () => {
+ beforeEach(() => {
+ postHogInitMock.mockClear();
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://eu.i.posthog.com");
+ });
+
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ });
+
+ it("never initializes PostHog in the HIPAA cloud region", async () => {
+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");
+
+ await importAppShell();
+
+ expect(postHogInitMock).not.toHaveBeenCalled();
+ });
+
+ it.each(ANALYTICS_ENABLED_REGIONS)(
+ "initializes PostHog in region %s",
+ async (region) => {
+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", region);
+
+ await importAppShell();
+
+ expect(postHogInitMock).toHaveBeenCalledTimes(1);
+ expect(postHogInitMock).toHaveBeenCalledWith(
+ "phc_test",
+ expect.objectContaining({ api_host: "https://eu.i.posthog.com" }),
+ );
+ },
+ );
+});
diff --git a/web/src/__tests__/posthog-session-replay-mask.clienttest.ts b/web/src/__tests__/posthog-session-replay-mask.clienttest.ts
index c7331fcf53..74e0aa87f9 100644
--- a/web/src/__tests__/posthog-session-replay-mask.clienttest.ts
+++ b/web/src/__tests__/posthog-session-replay-mask.clienttest.ts
@@ -42,16 +42,17 @@ describe("PostHog session replay privacy", () => {
expect(config.disable_session_recording).toBe(false);
});
- it("disables session recording in the HIPAA cloud region", async () => {
+ // Stronger than disabling the recorder: the HIPAA region runs no product
+ // analytics at all, so there is no PostHog client to record with. The wider
+ // region gate lives in posthog-product-analytics-region.clienttest.ts.
+ it("initializes no PostHog client in the HIPAA cloud region", async () => {
vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");
vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://us.i.posthog.com");
vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");
await import("@/src/pages/_app");
- expect(initMock).toHaveBeenCalledTimes(1);
- const config = initMock.mock.calls[0]![1] as Partial<PostHogConfig>;
- expect(config.disable_session_recording).toBe(true);
+ expect(initMock).not.toHaveBeenCalled();
});
it("disables session recording outside Langfuse Cloud", async () => {
diff --git a/web/src/__tests__/server/unit/serverPosthog.servertest.ts b/web/src/__tests__/server/unit/serverPosthog.servertest.ts
new file mode 100644
index 0000000000..d6547a5b95
--- /dev/null
+++ b/web/src/__tests__/server/unit/serverPosthog.servertest.ts
@@ -0,0 +1,104 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
+
+/**
+ * Product-analytics region gate (server half).
+ *
+ * `ServerPosthog` is the single server-side entry point for Langfuse product
+ * analytics (signup conversion, backend activity, self-host telemetry). In the
+ * HIPAA cloud region it still constructs a `posthog-node` client, then opts it
+ * out with `disable()` so capture becomes a no-op. That is enough on the
+ * server: unlike the browser SDK, disable does not phone home.
+ */
+
+const {
+ postHogConstructor,
+ captureMock,
+ disableMock,
+ shutdownMock,
+ flushMock,
+} = vi.hoisted(() => ({
+ postHogConstructor: vi.fn(),
+ captureMock: vi.fn(),
+ disableMock: vi.fn().mockResolvedValue(undefined),
+ shutdownMock: vi.fn().mockResolvedValue(undefined),
+ flushMock: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("posthog-node", () => ({
+ PostHog: class {
+ constructor(...args: unknown[]) {
+ postHogConstructor(...args);
+ }
+ capture = (...args: unknown[]) => {
+ if (disableMock.mock.calls.length > 0) return;
+ captureMock(...args);
+ };
+ disable = disableMock;
+ shutdown = shutdownMock;
+ flush = flushMock;
+ debug = vi.fn();
+ },
+}));
+
+const loadServerPosthog = async () => {
+ vi.resetModules();
+ const { ServerPosthog } =
+ await import("@/src/features/posthog-analytics/ServerPosthog");
+ return new ServerPosthog();
+};
+
+const captureArgs = {
+ distinctId: "user-1",
+ event: "backend:activity",
+} as const;
+
+describe("ServerPosthog product analytics region gate", () => {
+ beforeEach(() => {
+ postHogConstructor.mockClear();
+ captureMock.mockClear();
+ disableMock.mockClear();
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test");
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", "https://eu.i.posthog.com");
+ });
+
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ });
+
+ it("disables the client in the HIPAA cloud region so capture is a no-op", async () => {
+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");
+
+ const posthog = await loadServerPosthog();
+ posthog.capture(captureArgs);
+ await posthog.flush();
+ await posthog.shutdown();
+
+ expect(postHogConstructor).toHaveBeenCalledTimes(1);
+ expect(disableMock).toHaveBeenCalledTimes(1);
+ expect(captureMock).not.toHaveBeenCalled();
+ });
+
+ it("still disables the HIPAA client when it is built from the telemetry fallback", async () => {
+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", "HIPAA");
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", undefined);
+ vi.stubEnv("NEXT_PUBLIC_POSTHOG_HOST", undefined);
+
+ const posthog = await loadServerPosthog();
+ posthog.capture(captureArgs);
+
+ expect(postHogConstructor).toHaveBeenCalledTimes(1);
+ expect(disableMock).toHaveBeenCalledTimes(1);
+ expect(captureMock).not.toHaveBeenCalled();
+ });
+
+ it.each([["US"], ["EU"], ["JP"]])("keeps capturing in %s", async (region) => {
+ vi.stubEnv("NEXT_PUBLIC_LANGFUSE_CLOUD_REGION", region);
+
+ const posthog = await loadServerPosthog();
+ posthog.capture(captureArgs);
+
+ expect(postHogConstructor).toHaveBeenCalledTimes(1);
+ expect(disableMock).not.toHaveBeenCalled();
+ expect(captureMock).toHaveBeenCalledWith(captureArgs);
+ });
+});
diff --git a/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts b/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts
index 71b27ac745..1ca08bd673 100644
--- a/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts
+++ b/web/src/features/auth/lib/createProjectMembershipsOnSignup.ts
@@ -284,15 +284,13 @@ export async function createProjectMembershipsOnSignup(
// for conversion metric tracking in posthog: did a new user sign up?
// Fires on all production cloud regions, including ones added in the
// future. STAGING/DEV are excluded to keep test signups out of
- // conversion metrics, HIPAA is excluded deliberately to keep product
- // analytics off that deployment, and self-hosted deployments never emit
- // this event as the region env is unset.
+ // conversion metrics, and self-hosted deployments never emit this
+ // event as the region env is unset. HIPAA still constructs ServerPosthog
+ // here; the client is opted out via disable() so the capture is a no-op.
if (
isNewUser &&
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
- !["STAGING", "DEV", "HIPAA"].includes(
- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
- )
+ !["STAGING", "DEV"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)
) {
try {
const posthog = new ServerPosthog();
diff --git a/web/src/features/auth/lib/signOut.ts b/web/src/features/auth/lib/signOut.ts
index f310e8dd4d..a32634fac6 100644
--- a/web/src/features/auth/lib/signOut.ts
+++ b/web/src/features/auth/lib/signOut.ts
@@ -1,6 +1,7 @@
import { signOut } from "next-auth/react";
import posthog from "posthog-js";
import { env } from "@/src/env.mjs";
+import { isPostHogClientEnabled } from "@/src/features/posthog-analytics/productAnalyticsAvailability";
import { clearV4BetaEnabledSentryTag } from "@/src/utils/sentryV4BetaTag";
/**
@@ -19,7 +20,7 @@ export const signOutCleanly = async () => {
// see `unauthenticated`. Drop the pageload cache here so the sign-in
// hard load is not tagged with the previous user's v4 state.
clearV4BetaEnabledSentryTag();
- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {
+ if (isPostHogClientEnabled()) {
posthog.reset();
}
// On preview deployments the sign-in page signs visitors back in
diff --git a/web/src/features/posthog-analytics/ServerPosthog.ts b/web/src/features/posthog-analytics/ServerPosthog.ts
index 9162879417..10ea77f891 100644
--- a/web/src/features/posthog-analytics/ServerPosthog.ts
+++ b/web/src/features/posthog-analytics/ServerPosthog.ts
@@ -1,4 +1,5 @@
import { env } from "@/src/env.mjs";
+import { isProductAnalyticsAvailable } from "@/src/features/posthog-analytics/productAnalyticsAvailability";
import { PostHog } from "posthog-node";
const FALLBACK_POSTHOG_KEY = "phc_zkMwFajk8ehObUlMth0D7DtPItFnxETi3lmSvyQDrwB";
@@ -6,6 +7,7 @@ const FALLBACK_POSTHOG_HOST = "https://eu.posthog.com";
export class ServerPosthog {
private posthog: PostHog | null;
+ private optOut: Promise<void> | undefined;
constructor() {
const telemetryEnabled = env.TELEMETRY_ENABLED !== "false";
@@ -20,6 +22,13 @@ export class ServerPosthog {
if (apiKey && host) {
this.posthog = new PostHog(apiKey, { host });
if (process.env.NODE_ENV === "development") this.posthog.debug();
+ // Unlike the browser SDK, posthog-node disable() is a local opt-out:
+ // capture becomes a no-op and nothing is sent. HIPAA uses this instead
+ // of skipping construction. The flag flips synchronously; the promise is
+ // kept so the constructor does not float it.
+ if (!isProductAnalyticsAvailable()) {
+ this.optOut = this.posthog.disable();
+ }
} else {
this.posthog = null;
}
@@ -30,6 +39,7 @@ export class ServerPosthog {
}
async shutdown() {
+ await this.optOut;
await this.posthog?.shutdown();
}
diff --git a/web/src/features/posthog-analytics/productAnalyticsAvailability.ts b/web/src/features/posthog-analytics/productAnalyticsAvailability.ts
new file mode 100644
index 0000000000..daa1a40697
--- /dev/null
+++ b/web/src/features/posthog-analytics/productAnalyticsAvailability.ts
@@ -0,0 +1,23 @@
+// PostHog product analytics are off in the HIPAA cloud region. Not setting the
+// env vars is not enough: ServerPosthog falls back to Langfuse's telemetry key.
+//
+// Read from process.env, not the validated env object, so this also works at
+// module scope in the app shell where posthog-js is initialized. Next.js inlines
+// NEXT_PUBLIC_* per region build (see web/Dockerfile).
+
+export const isProductAnalyticsAvailable = (): boolean =>
+ process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== "HIPAA";
+
+/** Key/host for posthog-js, or null when the browser SDK must not initialize. */
+export const getPostHogClientConfig = (): {
+ key: string;
+ host: string;
+} | null => {
+ const key = process.env.NEXT_PUBLIC_POSTHOG_KEY;
+ const host = process.env.NEXT_PUBLIC_POSTHOG_HOST;
+ if (!isProductAnalyticsAvailable() || !key || !host) return null;
+ return { key, host };
+};
+
+export const isPostHogClientEnabled = (): boolean =>
+ getPostHogClientConfig() !== null;
diff --git a/web/src/pages/_app.tsx b/web/src/pages/_app.tsx
index 51cf036c78..f67c69a0f1 100644
--- a/web/src/pages/_app.tsx
+++ b/web/src/pages/_app.tsx
@@ -92,26 +92,31 @@ import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
import { ScoreCacheProvider } from "@/src/features/scores/contexts/ScoreCacheContext";
import { CorrectionCacheProvider } from "@/src/features/corrections/contexts/CorrectionCacheContext";
import { V4_BETA_ENABLED_POSTHOG_PROPERTY } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import {
+ getPostHogClientConfig,
+ isPostHogClientEnabled,
+ isProductAnalyticsAvailable,
+} from "@/src/features/posthog-analytics/productAnalyticsAvailability";
+// Session replay is a Langfuse Cloud feature, so self-hosted never records.
+// The product-analytics gate makes this redundant in HIPAA (PostHog is not
+// initialized there at all); it stays in the expression as defense in depth for
+// a compliance-sensitive flag.
const isPostHogSessionRecordingEnabled =
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined &&
- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== "HIPAA";
+ isProductAnalyticsAvailable();
-// Check that PostHog is client-side (used to handle Next.js SSR) and that env vars are set
-if (
- typeof window !== "undefined" &&
- process.env.NEXT_PUBLIC_POSTHOG_KEY &&
- process.env.NEXT_PUBLIC_POSTHOG_HOST
-) {
- posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY, {
- api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://eu.posthog.com",
+// Check that PostHog is client-side (used to handle Next.js SSR), that env vars
+// are set, and that the deployment region runs product analytics at all.
+const postHogClientConfig = getPostHogClientConfig();
+if (typeof window !== "undefined" && postHogClientConfig) {
+ posthog.init(postHogClientConfig.key, {
+ api_host: postHogClientConfig.host,
ui_host: "https://eu.posthog.com",
// Enable debug mode in development
loaded: (posthog) => {
if (process.env.NODE_ENV === "development") posthog.debug();
},
- // Session replay is a Langfuse Cloud feature and must stay disabled in the
- // HIPAA region. Other PostHog analytics remain independently configured.
disable_session_recording: !isPostHogSessionRecordingEnabled,
session_recording: {
maskAllInputs: true,
@@ -144,7 +149,7 @@ const MyApp: AppType<{ session: Session | null }> = ({
useEffect(() => {
// PostHog (cloud.langfuse.com)
- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {
+ if (isPostHogClientEnabled()) {
const handleRouteChange = () => {
posthog.capture("$pageview");
};
@@ -247,7 +252,7 @@ function UserTracking() {
) {
lastIdentifiedUser.current = JSON.stringify(sessionUser);
// PostHog
- if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {
+ if (isPostHogClientEnabled()) {
posthog.identify(sessionUser.id ?? undefined, {
environment: process.env.NODE_ENV,
email: sessionUser.email ?? undefined,