Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): name the srm segment cause for every reader (#127850)

Marcel Poelkere375fa85f2e9
diff --git a/products/experiments/backend/health/rollout.py b/products/experiments/backend/health/rollout.py
deleted file mode 100644
index 5e736f152ff3..000000000000
--- a/products/experiments/backend/health/rollout.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import posthoganalytics
-
-from posthog.clickhouse.query_tagging import get_query_tags
-from posthog.models.team.team import Team
-from posthog.models.user import User
-from posthog.utils import flag_identity
-
-# pinned: FEATURE_FLAGS.EXPERIMENT_HEALTH_FINDINGS in the frontend, which shows the findings.
-HEALTH_FINDINGS_FLAG = "experiment-health-findings"
-
-
-def runs_for_impersonated_session() -> bool:
- """
- Whether a staff member started this run in an impersonated session. A query runner has no session, so this reads
- the tag of the request, which follows an async run into the worker. The tag reads only the session cookie, so an
- OAuth token that was minted during impersonation is not an impersonated session here.
- """
- return bool(get_query_tags().is_impersonated)
-
-
-def shows_health_findings(user: User, team: Team) -> bool:
- """
- Whether the web app shows the health findings to the user in the team. An impersonated session always shows them,
- because the user here is the customer, whom the flag can leave out. Otherwise it evaluates the app's flag locally,
- from the person and group inputs that the app sends to /flags, so it never waits on the network. An evaluation
- that is undecided or fails is False.
- """
- if runs_for_impersonated_session():
- return True
- identity = flag_identity(user, team, team.organization)
- if identity is None:
- return False
- try:
- enabled = posthoganalytics.feature_enabled(
- HEALTH_FINDINGS_FLAG,
- identity.distinct_id,
- groups=identity.groups,
- person_properties=identity.person_properties,
- group_properties=identity.group_properties,
- only_evaluate_locally=True,
- send_feature_flag_events=False,
- )
- except Exception:
- return False
- return bool(enabled)
diff --git a/products/experiments/backend/health/test/test_rollout.py b/products/experiments/backend/health/test/test_rollout.py
deleted file mode 100644
index 58d2f45b82ff..000000000000
--- a/products/experiments/backend/health/test/test_rollout.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from contextlib import AbstractContextManager
-
-from posthog.test.base import BaseTest
-from unittest.mock import patch
-
-import posthoganalytics
-from parameterized import parameterized
-from posthoganalytics import Posthog
-
-from products.experiments.backend.health.rollout import HEALTH_FINDINGS_FLAG, shows_health_findings
-
-# The shape of a definition from /api/feature_flag/local_evaluation, with the rollout by email domain.
-HEALTH_FINDINGS_DEFINITION = {
- "id": 1,
- "key": HEALTH_FINDINGS_FLAG,
- "active": True,
- "deleted": False,
- "ensure_experience_continuity": False,
- "filters": {
- "groups": [
- {
- "properties": [{"key": "email", "type": "person", "value": "@example.com", "operator": "icontains"}],
- "rollout_percentage": 100,
- }
- ],
- "multivariate": None,
- },
-}
-
-
-class TestShowsHealthFindings(BaseTest):
- def setUp(self) -> None:
- super().setUp()
- self.client_with_definition = Posthog("phc_test", personal_api_key="phx_test", sync_mode=True)
- self.client_with_definition.feature_flags = [HEALTH_FINDINGS_DEFINITION]
- self.client_with_definition.group_type_mapping = {"0": "organization", "1": "project"}
-
- @parameterized.expand(
- [
- ("email_in_the_rollout", "[email protected]", False, True),
- ("email_outside_the_rollout", "[email protected]", False, False),
- ("anonymized", "[email protected]", True, False),
- ]
- )
- def test_evaluates_the_apps_flag_from_the_users_identity(
- self, _name: str, email: str, anonymize_data: bool, expected: bool
- ) -> None:
- self.user.email = email
- self.user.anonymize_data = anonymize_data
-
- with self._client_with_definition():
- self.assertEqual(shows_health_findings(self.user, self.team), expected)
-
- def test_shows_nothing_to_a_user_without_a_distinct_id(self) -> None:
- self.user.email = "[email protected]"
- self.user.distinct_id = None
-
- with self._client_with_definition():
- self.assertFalse(shows_health_findings(self.user, self.team))
-
- def _client_with_definition(self) -> AbstractContextManager[None]:
- # The module functions apply the module settings to the default client on every call, and the test settings
- # disable it.
- return patch.multiple(posthoganalytics, default_client=self.client_with_definition, disabled=False)
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 1a9044e223f7..9c545900e64a 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -68,7 +68,6 @@ from products.experiments.backend.health.registry import (
EXPOSURE_HEALTH_CHECKS,
evaluate as evaluate_health,
)
-from products.experiments.backend.health.rollout import runs_for_impersonated_session, shows_health_findings
from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY
from products.experiments.backend.hogql_queries.base_query_utils import analysis_window, analysis_window_end
from products.experiments.backend.hogql_queries.error_handling import experiment_error_handler
@@ -96,9 +95,9 @@ QUERY_ROW_LIMIT = 5000 # The result has one row per (day, variant) pair
# The queries that explain a health finding scan the exposure events. A timeout keeps the finding without the
# explanation, so it must not hold up the exposure answer as long as its own query can.
HEALTH_QUERY_TIMEOUT_SECONDS = 60
-# The page recalculates an experiment with few exposures on every entry, and the count without the test account filter
-# must not repeat its scan each time.
-EXPOSED_WITHOUT_FILTER_CACHE_SECONDS = 60 * 60
+# The page recalculates the exposures on a forced refresh, and on every entry while few users are exposed. The queries
+# that explain a finding must not repeat their scan each time.
+HEALTH_QUERY_CACHE_SECONDS = 60 * 60
def build_experiment_exposure_query(experiment: Experiment) -> ExperimentExposureQuery:
@@ -135,7 +134,6 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self.error_event_context = error_event_context
# The user id and whether that user may read the flag's history.
self._flag_history_access: tuple[int, bool] | None = None
- self._health_findings_shown: tuple[int, bool] | None = None
self._reads_precomputed_exposures = False
if not self.query.experiment_id:
@@ -332,17 +330,6 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self._flag_history_access = (self.user.pk, readable)
return self._flag_history_access[1]
- def _runs_segments_query(self) -> bool:
- # The segments only explain a finding and the query scans the exposure events, so it runs only for a reader
- # who sees the findings. A userless run sees none. The flag's answer stays out of the cache key: an async run
- # computes the key again in a worker, from that worker's flag definitions, and a key that disagrees with the web
- # request's makes every page load miss the cache.
- if not isinstance(self.user, User):
- return False
- if self._health_findings_shown is None or self._health_findings_shown[0] != self.user.pk:
- self._health_findings_shown = (self.user.pk, shows_health_findings(self.user, self.team))
- return self._health_findings_shown[1]
-
def _flag_history(self) -> FlagHistory | FlagHistoryNotChecked | None:
try:
restrictions = get_restricted_properties_with_group_type_index_for_team(
@@ -364,11 +351,12 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
context = load_health_context(self.experiment, exposures)
if reads_flag_history(context):
context = replace(context, flag_history=self._flag_history())
- if self._runs_segments_query() and reads_segments(context):
+ # These queries run for every reader, a userless run included. The findings depend on them, and the cache key
+ # doesn't say who computed the answer, so every reader with the same key shares it.
+ if reads_segments(context):
segments = self._segments(exposure_params, segments_first_exposed_after(context))
exposures = replace(exposures, segments=segments)
context = replace(context, exposures=exposures)
- # This count runs for every reader. The findings depend on it, and every reader shares the cached answer.
if exposures.filters_test_accounts and (
reads_exposed_without_test_account_filter(context) or reads_share_kept_by_test_account_filter(context)
):
@@ -379,10 +367,24 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
def _segments(
self, exposure_params: ExposureQueryParams, first_exposed_after: date | None
) -> tuple[ExposureSegment, ...] | None:
- query = self._query_builder(exposure_params).get_exposure_segments_query(
- TOP_SEGMENT_VALUES, first_exposed_after
+ cache_key = self._health_query_cache_key(
+ "exposure_segments",
+ {
+ "variants": self.variants,
+ "group_type_index": self.group_type_index,
+ "top_values": TOP_SEGMENT_VALUES,
+ "first_exposed_after": first_exposed_after.isoformat() if first_exposed_after else None,
+ },
)
- rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)
+ cached = get_safe_cache(cache_key)
+ if cached is not None:
+ rows = cached["rows"]
+ else:
+ query = self._query_builder(exposure_params).get_exposure_segments_query(
+ TOP_SEGMENT_VALUES, first_exposed_after
+ )
+ rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)
+ safe_cache_set(cache_key, {"rows": rows}, HEALTH_QUERY_CACHE_SECONDS)
if rows is None:
return None
exposed: dict[tuple[str, str, str], dict[str, int]] = {}
@@ -399,15 +401,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
)
def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:
- # A cache hit skips the masking of the query, so the key needs the reader's property access from the payload.
- cache_key = self._cache_key_for(
- {
- **self.get_cache_payload(),
- "exposed_without_test_account_filter": {
- "variants": self.variants,
- "group_type_index": self.group_type_index,
- },
- }
+ cache_key = self._health_query_cache_key(
+ "exposed_without_test_account_filter",
+ {"variants": self.variants, "group_type_index": self.group_type_index},
)
cached = get_safe_cache(cache_key)
if cached is not None:
@@ -416,9 +412,13 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
rows = self._run_health_query("ExperimentExposuresWithoutTestAccountFilterQuery", query)
exposed = None if rows is None else int(rows[0][0]) if rows else 0
# The cache keeps a failed count too, so a project that hits the timeout doesn't wait for it on every entry.
- safe_cache_set(cache_key, {"exposed": exposed}, EXPOSED_WITHOUT_FILTER_CACHE_SECONDS)
+ safe_cache_set(cache_key, {"exposed": exposed}, HEALTH_QUERY_CACHE_SECONDS)
return exposed
+ def _health_query_cache_key(self, name: str, inputs: dict[str, Any]) -> str:
+ # A cache hit skips the masking of the query, so the key needs the reader's property access from the payload.
+ return self._cache_key_for({**self.get_cache_payload(), name: inputs})
+
def _run_health_query(self, query_type: str, query: ast.SelectQuery) -> list[Any] | None:
"""The rows of a query that only explains a health finding. None when it fails, so that the finding keeps the
wording without the explanation and the exposure answer doesn't fail."""
@@ -618,8 +618,4 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
# The test account filter finding asks the user to edit these filters, so an edit must not serve the old answer.
if applies_test_account_filters(self.team, self.exposure_criteria):
payload["test_account_filters"] = self.team.test_account_filters
- # An impersonated session reads the segments whether or not the customer has the flag, so it must not get the
- # customer's cached answer without them. The query tag follows an async run, so a worker computes the same key.
- if runs_for_impersonated_session():
- payload["impersonated_session"] = True
return payload
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index 425746867fdf..1d23db7c25a1 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -15,7 +15,6 @@ from rest_framework.exceptions import ValidationError
from posthog.schema import (
ActionsNode,
- CachedExperimentExposureQueryResponse,
ExperimentEventExposureConfig,
ExperimentExposureCriteria,
ExperimentExposureHealthFindingCode,
@@ -27,15 +26,13 @@ from posthog.schema import (
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.query import execute_hogql_query
-from posthog.clickhouse.query_tagging import AccessMethod, reset_query_tags, tags_context
+from posthog.clickhouse.query_tagging import AccessMethod, tags_context
from posthog.constants import AvailableFeature
from posthog.exceptions import ClickHouseQueryTimeOut
-from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL
from posthog.models import PropertyDefinition
from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.user import User
-from posthog.tasks.tasks import process_query_task
from posthog.test.persons import create_group_type_mapping
from posthog.test.test_journeys import journeys_for
@@ -1631,10 +1628,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
flush_persons_and_events()
self._log_split_change_from_90_10_on_2024_01_03()
- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):
- response = ExperimentExposuresQueryRunner(
- team=self.team, query=self._running_exposure_query(), user=self.user
- ).calculate()
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
[finding] = response.health_findings or []
versions = sorted(
@@ -1673,72 +1669,10 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
created_at=datetime(2024, 1, 3, tzinfo=UTC),
)
- @parameterized.expand(
- [
- (
- "reader_who_sees_the_findings",
- True,
- 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '
- "library.",
- "library_version",
- {
- "located_dimensions": "library,library_version",
- "library_1": "web",
- "library_1_exposed_control": 100,
- "library_1_exposed_test": 100,
- "library_2": "posthog-node",
- "library_2_exposed_control": 5,
- "library_2_exposed_test": 120,
- "library_3": "posthog-ios",
- "library_3_exposed_control": 10,
- "library_3_exposed_test": 10,
- "library_version_1": "1.200.0",
- "library_version_1_library": "web",
- "library_version_1_exposed_control": 100,
- "library_version_1_exposed_test": 100,
- "library_version_2": "4.2.1",
- "library_version_2_library": "posthog-node",
- "library_version_2_exposed_control": 5,
- "library_version_2_exposed_test": 120,
- "library_version_3": "3.0.0",
- "library_version_3_library": "posthog-ios",
- "library_version_3_exposed_control": 10,
- "library_version_3_exposed_test": 10,
- "page_1": "/",
- "page_1_exposed_control": 34,
- "page_1_exposed_test": 34,
- "page_2": "/pricing",
- "page_2_exposed_control": 20,
- "page_2_exposed_test": 20,
- "page_3": "/docs/:page",
- "page_3_exposed_control": 15,
- "page_3_exposed_test": 15,
- "page_4": "/blog",
- "page_4_exposed_control": 12,
- "page_4_exposed_test": 12,
- "page_5": "/about",
- "page_5_exposed_control": 10,
- "page_5_exposed_test": 10,
- "page_6": BREAKDOWN_OTHER_STRING_LABEL,
- "page_6_exposed_control": 9,
- "page_6_exposed_test": 9,
- "screen_1": "Checkout",
- "screen_1_exposed_control": 10,
- "screen_1_exposed_test": 10,
- },
- ),
- (
- "reader_who_does_not_see_them",
- False,
- "This may indicate issues with randomization or data collection.",
- None,
- {},
- ),
- ]
- )
+ @parameterized.expand([("signed_in_reader", True), ("userless_run", False)])
@time_machine.travel("2024-01-07T12:00:00Z", tick=False)
def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(
- self, _name: str, findings_shown: bool, detail: str, reason: str | None, evidence: dict[str, str | int]
+ self, _name: str, as_user: bool
) -> None:
def exposure(variant: str, timestamp: str, library: str, version: str, **place: str | None) -> dict[str, Any]:
return {
@@ -1795,18 +1729,67 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
journeys_for({**web_first, **server_only, **mobile}, self.team)
flush_persons_and_events()
- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):
- response = ExperimentExposuresQueryRunner(
- team=self.team, query=self._running_exposure_query(), user=self.user
- ).calculate()
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None
+ ).calculate()
[finding] = response.health_findings or []
self.assertEqual(
- (finding.code, finding.subcode, finding.reason), (ExperimentExposureHealthFindingCode.SRM, None, reason)
+ (finding.code, finding.subcode, finding.reason),
+ (ExperimentExposureHealthFindingCode.SRM, None, "library_version"),
+ )
+ self.assertIn(
+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '
+ "library.",
+ finding.detail,
)
- self.assertIn(detail, finding.detail)
self.assertEqual(
- {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")}, evidence
+ {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")},
+ {
+ "located_dimensions": "library,library_version",
+ "library_1": "web",
+ "library_1_exposed_control": 100,
+ "library_1_exposed_test": 100,
+ "library_2": "posthog-node",
+ "library_2_exposed_control": 5,
+ "library_2_exposed_test": 120,
+ "library_3": "posthog-ios",
+ "library_3_exposed_control": 10,
+ "library_3_exposed_test": 10,
+ "library_version_1": "1.200.0",
+ "library_version_1_library": "web",
+ "library_version_1_exposed_control": 100,
+ "library_version_1_exposed_test": 100,
+ "library_version_2": "4.2.1",
+ "library_version_2_library": "posthog-node",
+ "library_version_2_exposed_control": 5,
+ "library_version_2_exposed_test": 120,
+ "library_version_3": "3.0.0",
+ "library_version_3_library": "posthog-ios",
+ "library_version_3_exposed_control": 10,
+ "library_version_3_exposed_test": 10,
+ "page_1": "/",
+ "page_1_exposed_control": 34,
+ "page_1_exposed_test": 34,
+ "page_2": "/pricing",
+ "page_2_exposed_control": 20,
+ "page_2_exposed_test": 20,
+ "page_3": "/docs/:page",
+ "page_3_exposed_control": 15,
+ "page_3_exposed_test": 15,
+ "page_4": "/blog",
+ "page_4_exposed_control": 12,
+ "page_4_exposed_test": 12,
+ "page_5": "/about",
+ "page_5_exposed_control": 10,
+ "page_5_exposed_test": 10,
+ "page_6": BREAKDOWN_OTHER_STRING_LABEL,
+ "page_6_exposed_control": 9,
+ "page_6_exposed_test": 9,
+ "screen_1": "Checkout",
+ "screen_1_exposed_control": 10,
+ "screen_1_exposed_test": 10,
+ },
)
@parameterized.expand(
@@ -1870,10 +1853,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
)
flush_persons_and_events()
- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):
- response = ExperimentExposuresQueryRunner(
- team=self.team, query=self._running_exposure_query(), user=self.user
- ).calculate()
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
[finding] = [
finding
@@ -1928,20 +1910,26 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
runner_module = ExperimentExposuresQueryRunner.__module__
with (
patch(f"{runner_module}.execute_hogql_query", side_effect=execute),
- patch(f"{runner_module}.shows_health_findings", return_value=True),
patch(f"{runner_module}.logger") as logger,
patch(f"{runner_module}.capture_exception") as capture_exception,
):
- response = ExperimentExposuresQueryRunner(
- team=self.team, query=self._running_exposure_query(), user=self.user
- ).calculate()
+ query = self._running_exposure_query()
+ responses = [
+ ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).calculate()
+ for _ in range(2)
+ ]
- self.assertIn("ExperimentExposureSegmentsQuery", query_types)
+ self.assertEqual(query_types.count("ExperimentExposureSegmentsQuery"), 1)
self.assertEqual((logger.warning.called, capture_exception.called), (logged, not logged))
- self.assertEqual(response.total_exposures, {"control": 90, "test": 10})
+ self.assertEqual([response.total_exposures for response in responses], [{"control": 90, "test": 10}] * 2)
self.assertEqual(
- [(finding.code, finding.detail, set(finding.evidence)) for finding in response.health_findings or []],
[
+ (finding.code, finding.detail, set(finding.evidence))
+ for response in responses
+ for finding in response.health_findings or []
+ ],
+ 2
+ * [
(
ExperimentExposureHealthFindingCode.SRM,
"The distribution of users across variants doesn't match your configured rollout percentages "
@@ -1951,81 +1939,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
],
)
- def _expose_a_mismatch_located_in_one_library_version(self) -> None:
- ff_property = f"$feature/{self.feature_flag.key}"
- journeys_for(
- {
- f"user_{library}_{variant}_{i}": [
- {
- "event": "$feature_flag_called",
- "timestamp": "2024-01-02",
- "properties": {
- "$feature_flag_response": variant,
- ff_property: variant,
- "$feature_flag": self.feature_flag.key,
- "$lib": library,
- "$lib_version": "1.0.0",
- },
- }
- ]
- for library, counts in (
- ("web", (("control", 60), ("test", 60))),
- ("posthog-node", (("control", 15), ("test", 105))),
- )
- for variant, count in counts
- for i in range(count)
- },
- self.team,
- )
- flush_persons_and_events()
-
- @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
- def test_impersonated_session_reads_the_segments_and_keeps_its_answer_apart_from_the_customers(self) -> None:
- self._expose_a_mismatch_located_in_one_library_version()
- query = self._running_exposure_query()
-
- def cached_and_reason() -> tuple[bool | None, str | None]:
- response = ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).run(
- ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE
- )
- assert isinstance(response, CachedExperimentExposureQueryResponse)
- [finding] = response.health_findings or []
- return response.is_cached, finding.reason
-
- customer = cached_and_reason()
- with tags_context(is_impersonated=True):
- impersonated = cached_and_reason()
- customer_again = cached_and_reason()
-
- self.assertEqual(
- [customer, impersonated, customer_again],
- [(False, None), (False, "library_version"), (True, None)],
- )
-
- @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
- def test_async_run_of_an_impersonated_session_fills_the_entry_the_web_request_reads(self) -> None:
- self._expose_a_mismatch_located_in_one_library_version()
- query = self._running_exposure_query()
-
- def run(execution_mode: ExecutionMode) -> Any:
- return ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).run(
- execution_mode, user=self.user
- )
-
- with tags_context(is_impersonated=True), patch("posthog.tasks.tasks.process_query_task.si") as queued_task:
- run(ExecutionMode.CALCULATE_ASYNC_ALWAYS)
-
- # A worker starts without the tags of the request, so only what the queued task carries reaches it.
- with tags_context():
- reset_query_tags()
- process_query_task.apply(args=queued_task.call_args.args, kwargs=queued_task.call_args.kwargs).get()
-
- with tags_context(is_impersonated=True):
- response = run(ExecutionMode.CACHE_ONLY_NEVER_CALCULATE)
-
- [finding] = response.health_findings or []
- self.assertEqual(finding.reason, "library_version")
-
def test_cache_key_keeps_the_flag_history_from_a_reader_who_cannot_view_the_flag(self) -> None:
self.organization.available_product_features = [
{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
@@ -2116,7 +2029,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
0,
2,
LEAVES_OUT_TEST_ACCOUNTS,
- True,
"2024-01-07T12:00:00Z",
[
(
@@ -2132,7 +2044,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
0,
0,
LEAVES_OUT_TEST_ACCOUNTS,
- True,
"2024-01-07T12:00:00Z",
[
(
@@ -2143,44 +2054,11 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
)
],
),
- (
- "test_accounts_exposed_for_a_reader_who_does_not_see_the_findings",
- 0,
- 2,
- LEAVES_OUT_TEST_ACCOUNTS,
- False,
- "2024-01-07T12:00:00Z",
- [
- (
- ExperimentExposureHealthFindingCode.ZERO_EXPOSURES,
- "calls_not_counted",
- "No users exposed: the test account filter removes all of them",
- 2,
- )
- ],
- ),
- (
- "few_users_counted_for_a_reader_who_does_not_see_the_findings",
- 3,
- 100,
- KEEPS_MATCHING_USERS,
- False,
- "2024-01-07T12:00:00Z",
- [
- (
- ExperimentExposureHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
- None,
- "The test account filter removes most exposed users",
- 103,
- )
- ],
- ),
(
"few_users_counted",
3,
100,
KEEPS_MATCHING_USERS,
- True,
"2024-01-07T12:00:00Z",
[
(
@@ -2191,9 +2069,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
)
],
),
- ("few_users_counted_in_the_first_day", 3, 100, KEEPS_MATCHING_USERS, True, "2024-01-01T23:00:00Z", []),
- ("few_users_counted_with_only_negative_filters", 3, 100, LEAVES_OUT_TEST_ACCOUNTS, True, LATER, []),
- ("few_users_counted_with_only_a_hogql_filter", 3, 100, HOGQL_FILTER, True, LATER, []),
+ ("few_users_counted_in_the_first_day", 3, 100, KEEPS_MATCHING_USERS, "2024-01-01T23:00:00Z", []),
+ ("few_users_counted_with_only_negative_filters", 3, 100, LEAVES_OUT_TEST_ACCOUNTS, LATER, []),
+ ("few_users_counted_with_only_a_hogql_filter", 3, 100, HOGQL_FILTER, LATER, []),
]
)
def test_counts_the_users_that_the_test_account_filter_removes(
@@ -2202,7 +2080,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
users: int,
test_accounts: int,
test_account_filter: dict[str, Any],
- findings_shown: bool,
now: str,
expected: list[tuple[ExperimentExposureHealthFindingCode, str | None, str, int | None]],
) -> None:
@@ -2217,7 +2094,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
with (
time_machine.travel(now, tick=False),
patch(f"{runner_module}.execute_hogql_query", side_effect=execute),
- patch(f"{runner_module}.shows_health_findings", return_value=findings_shown),
):
response = ExperimentExposuresQueryRunner(
team=self.team, query=self._running_exposure_query(), user=self.user