Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(frontend): send boot timing for every page load in the boot experiment (#124799)

Paul D'Ambraa54bfb623c1f
diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx
index 4baccde410dd..bf2c22b4f4f3 100644
--- a/frontend/src/index.tsx
+++ b/frontend/src/index.tsx
@@ -12,7 +12,10 @@ import { retryBootImport } from 'lib/utils/retryImport'
import { RootErrorBoundary } from './RootErrorBoundary'
import { ChunkLoadErrorBoundary } from './scenes/ChunkLoadErrorBoundary'
-markBootPhase('entry', { deferred_app_context: !!window.POSTHOG_APP_CONTEXT_REQUEST })
+markBootPhase('entry', {
+ deferred_app_context: !!window.POSTHOG_APP_CONTEXT_REQUEST,
+ deferred_app_context_variant: window.POSTHOG_APP_CONTEXT?.deferred_app_context_variant,
+})
type AppModules = [typeof import('scenes/App'), typeof import('scenes/bootApp')]
diff --git a/frontend/src/lib/utils/bootTiming.test.ts b/frontend/src/lib/utils/bootTiming.test.ts
index 674ad2c84046..221ffb0d4267 100644
--- a/frontend/src/lib/utils/bootTiming.test.ts
+++ b/frontend/src/lib/utils/bootTiming.test.ts
@@ -138,19 +138,31 @@ describe('bootTiming', () => {
})
it.each([
- { distinctId: IN_SAMPLE_ID, sends: true },
- { distinctId: OUT_OF_SAMPLE_ID, sends: false },
+ { distinctId: IN_SAMPLE_ID, variant: undefined, sampleRate: 0.05 },
+ { distinctId: OUT_OF_SAMPLE_ID, variant: undefined, sampleRate: null },
// An empty ID hashes to bucket 0, so without a guard every such page load would land in the sample.
- { distinctId: '', sends: false },
- { distinctId: undefined, sends: false },
- ])('samples on distinct ID $distinctId (sends: $sends)', async ({ distinctId, sends }) => {
- getDistinctId.mockReturnValue(distinctId)
- markAt(50, 'entry')
- markAt(900, 'scene_rendered')
- await flushSend()
+ { distinctId: '', variant: undefined, sampleRate: null },
+ { distinctId: undefined, variant: undefined, sampleRate: null },
+ { distinctId: OUT_OF_SAMPLE_ID, variant: 'control', sampleRate: 1 },
+ { distinctId: OUT_OF_SAMPLE_ID, variant: 'test', sampleRate: 1 },
+ ])(
+ 'samples on distinct ID $distinctId in variant $variant (sample rate: $sampleRate)',
+ async ({ distinctId, variant, sampleRate }) => {
+ getDistinctId.mockReturnValue(distinctId)
+ markAt(50, 'entry', { deferred_app_context_variant: variant })
+ markAt(900, 'scene_rendered')
+ await flushSend()
- expect(capture).toHaveBeenCalledTimes(sends ? 1 : 0)
- })
+ if (sampleRate === null) {
+ expect(capture).not.toHaveBeenCalled()
+ } else {
+ expect(capture).toHaveBeenCalledWith(
+ 'app boot timing',
+ expect.objectContaining({ sample_rate: sampleRate })
+ )
+ }
+ }
+ )
it.each([
{
diff --git a/frontend/src/lib/utils/bootTiming.ts b/frontend/src/lib/utils/bootTiming.ts
index b25cb14dc2aa..572016ce4370 100644
--- a/frontend/src/lib/utils/bootTiming.ts
+++ b/frontend/src/lib/utils/bootTiming.ts
@@ -17,6 +17,7 @@ export interface BootContext {
scene?: string
logged_in?: boolean
deferred_app_context?: boolean
+ deferred_app_context_variant?: string
}
const phaseTimes: Partial<Record<BootPhase, number>> = {}
@@ -62,8 +63,10 @@ async function sendBootTiming(): Promise<void> {
// Not `$device_id`: posthog-js gives it a new value on each page load of a user bootstrapped with
// `identity_distinct_id`. The distinct ID changes only at login, so a user stays in or out across page loads.
const distinctId = posthog.get_distinct_id()
+ // An experiment on boot time needs every page load of its users, not the 5% sample.
+ const sampleRate = context.deferred_app_context_variant ? 1 : SAMPLE_RATE
// hashCodeForString returns 0 for an empty string, which would put every such page load in the sample.
- if (!distinctId?.trim() || hashCodeForString(distinctId) % 100 >= SAMPLE_RATE * 100) {
+ if (!distinctId?.trim() || hashCodeForString(distinctId) % 100 >= sampleRate * 100) {
return
}
const durations = Object.fromEntries(
@@ -74,7 +77,7 @@ async function sendBootTiming(): Promise<void> {
...context,
...getNavigationProperties(),
page_was_hidden: pageWasHidden,
- sample_rate: SAMPLE_RATE,
+ sample_rate: sampleRate,
})
}
diff --git a/frontend/src/lib/utils/deferredAppContext.ts b/frontend/src/lib/utils/deferredAppContext.ts
index 0c6bd7cd08ca..3c87ce27aa5b 100644
--- a/frontend/src/lib/utils/deferredAppContext.ts
+++ b/frontend/src/lib/utils/deferredAppContext.ts
@@ -2,7 +2,7 @@ import type { AppContext } from '~/types'
import { markBootPhase } from './bootTiming'
-// pinned: query param read by `should_defer_app_context` in posthog/utils.py
+// pinned: query param read by `deferred_app_context_variant` in posthog/utils.py
const DEFERRED_APP_CONTEXT_PARAM = 'deferred_app_context'
interface DeferredAppContext {
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 08697061629c..d4b810e4efab 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -5341,6 +5341,8 @@ export type EventOrPropType = EventDefinition & PropertyDefinition
export interface AppContext {
/** The page left out the user's data, and the app fetches it at boot. See lib/utils/deferredAppContext. */
deferred?: boolean
+ /** The user's variant in the deferred app context experiment. Absent outside the experiment. */
+ deferred_app_context_variant?: string
run_mode?: 'US' | 'EU' | 'DEV' | 'E2E' | 'LOCAL' | 'HOBBY'
current_user: UserType | null
current_project: ProjectType | null
diff --git a/posthog/test/test_get_context_for_template.py b/posthog/test/test_get_context_for_template.py
index 3b2673f00a74..c9af790aab55 100644
--- a/posthog/test/test_get_context_for_template.py
+++ b/posthog/test/test_get_context_for_template.py
@@ -21,10 +21,10 @@ from posthog.stable_chunks import StableChunks
from posthog.templatetags.posthog_filters import TolerantJSONEncoder
from posthog.utils import (
DEFERRED_APP_CONTEXT_FLAG,
+ deferred_app_context_variant,
flags_for_app_evaluation_context,
get_context_for_template,
get_persisted_feature_flags_for_app_context,
- should_defer_app_context,
)
from products.conversations.backend.services.identity import IDENTITY_CLAIM_MAX_AGE_SECONDS
@@ -132,14 +132,14 @@ class TestFlagsForAppEvaluationContext(SimpleTestCase):
class TestShouldDeferAppContext(SimpleTestCase):
@parameterized.expand(
[
- ("test variant defers and reports", "", "test", True, True),
- ("control variant renders inline and reports", "", "control", False, True),
- ("outside the experiment renders inline silently", "", None, False, False),
- ("param on beats control", "?deferred_app_context=1", "control", True, False),
- ("param off beats test", "?deferred_app_context=0", "test", False, False),
+ ("test variant reports", "", "test", "test", True),
+ ("control variant reports", "", "control", "control", True),
+ ("outside the experiment is silent", "", None, None, False),
+ ("param on beats control", "?deferred_app_context=1", "control", "test", False),
+ ("param off beats test", "?deferred_app_context=0", "test", "control", False),
]
)
- def test_only_the_test_variant_defers(self, _name, query, variant, expect_deferred, expect_exposure):
+ def test_picks_the_variant(self, _name, query, variant, expected_variant, expect_exposure):
request = RequestFactory().get(f"/{query}")
user = User(distinct_id="user-1", email="[email protected]")
@@ -147,7 +147,7 @@ class TestShouldDeferAppContext(SimpleTestCase):
mock.patch("posthog.utils.posthoganalytics.get_feature_flag", return_value=variant),
mock.patch("posthog.utils.posthoganalytics.capture") as capture,
):
- assert should_defer_app_context(request, user) is expect_deferred
+ assert deferred_app_context_variant(request, user) == expected_variant
if expect_exposure:
capture.assert_called_once_with(
@@ -410,6 +410,8 @@ class TestGetContextForTemplate(APIBaseTest):
assert deferred["posthog_app_context"]["deferred"] is True
assert deferred["posthog_app_context"]["current_user"] is None
assert "featureFlags" not in deferred["posthog_bootstrap"]
+ assert deferred["posthog_app_context"].pop("deferred_app_context_variant") == "test"
+ assert inline["posthog_app_context"].pop("deferred_app_context_variant") == "control"
page_with_fetched = {**deferred["posthog_app_context"], **fetched["app_context"]}
page_with_fetched.pop("deferred")
assert page_with_fetched == json.loads(json.dumps(inline["posthog_app_context"], cls=TolerantJSONEncoder))
diff --git a/posthog/utils.py b/posthog/utils.py
index acc066b64fed..be7e79818099 100644
--- a/posthog/utils.py
+++ b/posthog/utils.py
@@ -704,7 +704,10 @@ def _build_template_context(
).data
elif request.user.pk:
user = cast("User", request.user)
- if template_name == "index.html" and should_defer_app_context(request, user):
+ variant = deferred_app_context_variant(request, user) if template_name == "index.html" else None
+ if variant:
+ posthog_app_context["deferred_app_context_variant"] = variant
+ if variant == "test":
# The page fetches the user's data from `app_context` while its JS loads.
posthog_app_context["deferred"] = True
posthog_distinct_id = user.distinct_id
@@ -813,14 +816,15 @@ DEFERRED_APP_CONTEXT_FLAG = "deferred-app-context"
DEFERRED_APP_CONTEXT_PARAM = "deferred_app_context"
-def should_defer_app_context(request: HttpRequest, user: "User") -> bool:
+def deferred_app_context_variant(request: HttpRequest, user: "User") -> Optional[str]:
"""
- Whether the app page leaves out the user's data and fetches it from `app_context` instead.
+ The variant of the app page. "test" leaves out the user's data and fetches it from `app_context` instead.
+ None means the user is outside the experiment.
The query param wins over the experiment, so a lab run can pick a side and a failed fetch can reload without it.
"""
param = request.GET.get(DEFERRED_APP_CONTEXT_PARAM)
if param is not None:
- return param == "1"
+ return "test" if param == "1" else "control"
try:
variant = posthoganalytics.get_feature_flag(
DEFERRED_APP_CONTEXT_FLAG,
@@ -836,10 +840,11 @@ def should_defer_app_context(request: HttpRequest, user: "User") -> bool:
distinct_id=str(user.distinct_id),
properties={"$feature_flag": DEFERRED_APP_CONTEXT_FLAG, "$feature_flag_response": variant},
)
- return variant == "test"
+ return variant
+ return None
except Exception as e:
capture_exception(e)
- return False
+ return None
def feature_flag_bootstrap(user: "User", current_user: dict[str, Any]) -> Optional[dict[str, Any]]: